{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:04.802532Z",
     "iopub.status.busy": "2023-11-07T10:37:04.802188Z",
     "iopub.status.idle": "2023-11-07T10:37:05.158789Z",
     "shell.execute_reply": "2023-11-07T10:37:05.157776Z",
     "shell.execute_reply.started": "2023-11-07T10:37:04.802504Z"
    }
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import gc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:05.160810Z",
     "iopub.status.busy": "2023-11-07T10:37:05.160393Z",
     "iopub.status.idle": "2023-11-07T10:37:08.715694Z",
     "shell.execute_reply": "2023-11-07T10:37:08.714630Z",
     "shell.execute_reply.started": "2023-11-07T10:37:05.160783Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from torch.optim import AdamW\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils import clip_grad_norm_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:08.717306Z",
     "iopub.status.busy": "2023-11-07T10:37:08.716874Z",
     "iopub.status.idle": "2023-11-07T10:37:18.250810Z",
     "shell.execute_reply": "2023-11-07T10:37:18.249788Z",
     "shell.execute_reply.started": "2023-11-07T10:37:08.717279Z"
    }
   },
   "outputs": [],
   "source": [
    "import transformers\n",
    "from transformers import BertForSequenceClassification, BertTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:18.253259Z",
     "iopub.status.busy": "2023-11-07T10:37:18.252965Z",
     "iopub.status.idle": "2023-11-07T10:37:18.623596Z",
     "shell.execute_reply": "2023-11-07T10:37:18.622717Z",
     "shell.execute_reply.started": "2023-11-07T10:37:18.253233Z"
    }
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:18.625607Z",
     "iopub.status.busy": "2023-11-07T10:37:18.624902Z",
     "iopub.status.idle": "2023-11-07T10:37:18.631360Z",
     "shell.execute_reply": "2023-11-07T10:37:18.630186Z",
     "shell.execute_reply.started": "2023-11-07T10:37:18.625568Z"
    }
   },
   "outputs": [],
   "source": [
    "def free_memory():\n",
    "    gc.collect()\n",
    "    torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:18.633005Z",
     "iopub.status.busy": "2023-11-07T10:37:18.632698Z",
     "iopub.status.idle": "2023-11-07T10:37:19.463152Z",
     "shell.execute_reply": "2023-11-07T10:37:19.462135Z",
     "shell.execute_reply.started": "2023-11-07T10:37:18.632979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>качество плохое пошив ужасный (горловина напер...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Товар отдали другому человеку, я не получила п...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ужасная синтетика! Тонкая, ничего общего с пре...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>товар не пришел, продавец продлил защиту без м...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Кофточка голая синтетика, носить не возможно.</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Очень глубокие проймы</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Я недовольна заказом.Я вот одного не понимаю п...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>заказала размер s на от 64,об 94,начнем с того...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Заказ я сделала в июле. С тех пор посылка отсл...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Ужасное качество товара!</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  качество плохое пошив ужасный (горловина напер...  negative\n",
       "1  Товар отдали другому человеку, я не получила п...  negative\n",
       "2  Ужасная синтетика! Тонкая, ничего общего с пре...  negative\n",
       "3  товар не пришел, продавец продлил защиту без м...  negative\n",
       "4      Кофточка голая синтетика, носить не возможно.  negative\n",
       "5                              Очень глубокие проймы  negative\n",
       "6  Я недовольна заказом.Я вот одного не понимаю п...  negative\n",
       "7  заказала размер s на от 64,об 94,начнем с того...  negative\n",
       "8  Заказ я сделала в июле. С тех пор посылка отсл...  negative\n",
       "9                           Ужасное качество товара!  negative"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews = pd.read_csv('data/reviews.csv.zip', sep='\\t')\n",
    "reviews.drop_duplicates(subset=['review'], inplace=True)\n",
    "reviews.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:19.465013Z",
     "iopub.status.busy": "2023-11-07T10:37:19.464633Z",
     "iopub.status.idle": "2023-11-07T10:37:19.769137Z",
     "shell.execute_reply": "2023-11-07T10:37:19.768248Z",
     "shell.execute_reply.started": "2023-11-07T10:37:19.464979Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAj4AAAGdCAYAAAASUnlxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAA0bUlEQVR4nO3de3xNd77/8XcSyU6CHdckMoKcapHWXbH1pkQ2Mk61pocyLQY9PJK25Lg0c5SgfaQ11dLSmjmdis5hRs1MdYqS3RhUpS4hbi2jhjHzYEfrFqGSSNbvj/6yjl2XSuxI5ft6Ph55PKy1Puu7vmvZ35131mXvAMuyLAEAABggsLo7AAAAcKsQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxqhV3R2oTmVlZTp27Jjq1q2rgICA6u4OAAC4AZZl6dy5c4qJiVFgYMXO4RgdfI4dO6bY2Njq7gYAAKiEf/7zn2ratGmF1jE6+NStW1fSdwfO6XT6rd2SkhJlZWUpMTFRwcHBfmsXwI1jHALVqyrHYEFBgWJjY+3f4xVhdPApv7zldDr9HnzCw8PldDp5wwWqCeMQqF63YgxW5jYVbm4GAADGIPgAAABjVCj4vP3222rXrp19acjlcunjjz+2l1+8eFHJyclq2LCh6tSpo0GDBik/P9+njaNHjyopKUnh4eGKjIzUpEmTdOnSJZ+a9evXq1OnTnI4HGrZsqUyMzOv6MuCBQvUokULhYaGqlu3btq6dWtFdgUAABioQsGnadOmevnll5Wbm6vt27erV69eeuSRR7Rv3z5J0oQJE/TRRx9p+fLl2rBhg44dO6bHHnvMXr+0tFRJSUkqLi7W5s2btXjxYmVmZmratGl2zeHDh5WUlKSHH35YeXl5Gj9+vEaPHq21a9faNcuWLVNqaqqmT5+uHTt2qH379nK73Tpx4sTNHg8AAFCTWTepfv361jvvvGOdOXPGCg4OtpYvX24v+/LLLy1JVk5OjmVZlrV69WorMDDQ8nq9ds3bb79tOZ1Oq6ioyLIsy5o8ebJ19913+2xj8ODBltvttqe7du1qJScn29OlpaVWTEyMlZGRUaG+nz171pJknT17tkLr/ZDi4mJrxYoVVnFxsV/bBXDjGIdA9arKMXgzv78r/VRXaWmpli9frvPnz8vlcik3N1clJSVKSEiwa1q3bq1mzZopJydH3bt3V05Ojtq2bauoqCi7xu12a9y4cdq3b586duyonJwcnzbKa8aPHy9JKi4uVm5urtLS0uzlgYGBSkhIUE5OznX7XFRUpKKiInu6oKBA0nd3npeUlFT2UFyhvC1/tgmgYhiHQPWqyjF4M21WOPjs2bNHLpdLFy9eVJ06dfTBBx8oPj5eeXl5CgkJUb169Xzqo6Ki5PV6JUler9cn9JQvL192vZqCggJ9++23On36tEpLS69as3///uv2PSMjQzNmzLhiflZWlsLDw3945yvI4/H4vU0AFcM4BKpXVYzBCxcuVHrdCgefVq1aKS8vT2fPntUf//hHDR8+XBs2bKh0B26ltLQ0paam2tPlH4CUmJjo98/x8Xg86tOnD58fAlQTxiFQvapyDJZfsamMCgefkJAQtWzZUpLUuXNnbdu2TfPmzdPgwYNVXFysM2fO+Jz1yc/PV3R0tCQpOjr6iqevyp/6urzm+0+C5efny+l0KiwsTEFBQQoKCrpqTXkb1+JwOORwOK6YHxwcXCVvjFXVLoAbxzgEqldVjMGbae+mP8enrKxMRUVF6ty5s4KDg5WdnW0vO3DggI4ePSqXyyVJcrlc2rNnj8/TVx6PR06nU/Hx8XbN5W2U15S3ERISos6dO/vUlJWVKTs7264BAAC4mgqd8UlLS1O/fv3UrFkznTt3TkuXLtX69eu1du1aRUREaNSoUUpNTVWDBg3kdDr1zDPPyOVyqXv37pKkxMRExcfH68knn9Ts2bPl9Xo1depUJScn22dixo4dq/nz52vy5Mn6xS9+oXXr1un999/XqlWr7H6kpqZq+PDh6tKli7p27aq5c+fq/PnzGjlypB8PDQAAqGkqFHxOnDihp556SsePH1dERITatWuntWvXqk+fPpKk119/XYGBgRo0aJCKiorkdrv11ltv2esHBQVp5cqVGjdunFwul2rXrq3hw4dr5syZdk1cXJxWrVqlCRMmaN68eWratKneeecdud1uu2bw4MH6+uuvNW3aNHm9XnXo0EFr1qy54oZnAACAywVYlmVVdyeqS0FBgSIiInT27Fm/39y8evVq9e/fn3sLgGrCOASqV1WOwZv5/c13dQEAAGNU+gMMAQDArdPi+VU/XPQj4giyNLtrdffiSgSfKnRP+loVlQZUdzdu2JGXk6q7CwAAVCkudQEAAGMQfAAAgDEIPgAAwBgEHwAAYAxubgZQo/GQAYDLccYHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGKNCwScjI0P33nuv6tatq8jISA0cOFAHDhzwqenZs6cCAgJ8fsaOHetTc/ToUSUlJSk8PFyRkZGaNGmSLl265FOzfv16derUSQ6HQy1btlRmZuYV/VmwYIFatGih0NBQdevWTVu3bq3I7gAAAMNUKPhs2LBBycnJ+vzzz+XxeFRSUqLExESdP3/ep27MmDE6fvy4/TN79mx7WWlpqZKSklRcXKzNmzdr8eLFyszM1LRp0+yaw4cPKykpSQ8//LDy8vI0fvx4jR49WmvXrrVrli1bptTUVE2fPl07duxQ+/bt5Xa7deLEicoeCwAAUMPVqkjxmjVrfKYzMzMVGRmp3NxcPfjgg/b88PBwRUdHX7WNrKwsffHFF/rkk08UFRWlDh06aNasWZoyZYrS09MVEhKihQsXKi4uTnPmzJEktWnTRps2bdLrr78ut9stSXrttdc0ZswYjRw5UpK0cOFCrVq1Su+++66ef/75iuwWAAAwRIWCz/edPXtWktSgQQOf+UuWLNH//u//Kjo6WgMGDNALL7yg8PBwSVJOTo7atm2rqKgou97tdmvcuHHat2+fOnbsqJycHCUkJPi06Xa7NX78eElScXGxcnNzlZaWZi8PDAxUQkKCcnJyrtnfoqIiFRUV2dMFBQWSpJKSEpWUlFTiCFxdeVuOQMtvbd4K/jwGQHVjHKKmcQTdXq/l8rFXFa/pm2mz0sGnrKxM48eP13333ad77rnHnj906FA1b95cMTEx2r17t6ZMmaIDBw7oz3/+syTJ6/X6hB5J9rTX671uTUFBgb799ludPn1apaWlV63Zv3//NfuckZGhGTNmXDE/KyvLDmb+NKtLmd/brEqrV6+u7i4Afsc4RE0xu2t196ByPB6P39u8cOFCpdetdPBJTk7W3r17tWnTJp/5Tz/9tP3vtm3bqkmTJurdu7cOHTqkO+64o9Id9Ye0tDSlpqba0wUFBYqNjVViYqKcTqfftlNSUiKPx6MXtgeqqCzAb+1Wtb3p7uruAuA3jEPUNPekr/3hoh8RR6ClWV3K1KdPHwUHB/u17fIrNpVRqeCTkpKilStXauPGjWratOl1a7t16yZJ+uqrr3THHXcoOjr6iqev8vPzJcm+Lyg6Otqed3mN0+lUWFiYgoKCFBQUdNWaa91bJEkOh0MOh+OK+cHBwX7/T5GkorIAFZXePm+4VXEMgOrGOERNcTu9ji9XFb9jb6a9Cj3VZVmWUlJS9MEHH2jdunWKi4v7wXXy8vIkSU2aNJEkuVwu7dmzx+fpK4/HI6fTqfj4eLsmOzvbpx2PxyOXyyVJCgkJUefOnX1qysrKlJ2dbdcAAAB8X4XO+CQnJ2vp0qX68MMPVbduXfuenIiICIWFhenQoUNaunSp+vfvr4YNG2r37t2aMGGCHnzwQbVr106SlJiYqPj4eD355JOaPXu2vF6vpk6dquTkZPtszNixYzV//nxNnjxZv/jFL7Ru3Tq9//77WrVqld2X1NRUDR8+XF26dFHXrl01d+5cnT9/3n7KCwAA4PsqFHzefvttSd99SOHlFi1apBEjRigkJESffPKJHUJiY2M1aNAgTZ061a4NCgrSypUrNW7cOLlcLtWuXVvDhw/XzJkz7Zq4uDitWrVKEyZM0Lx589S0aVO988479qPskjR48GB9/fXXmjZtmrxerzp06KA1a9ZcccMzAABAuQoFH8u6/qN0sbGx2rBhww+207x58x98cqFnz57auXPndWtSUlKUkpLyg9sDAACQ+K4uAABgEIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgjAoFn4yMDN17772qW7euIiMjNXDgQB04cMCn5uLFi0pOTlbDhg1Vp04dDRo0SPn5+T41R48eVVJSksLDwxUZGalJkybp0qVLPjXr169Xp06d5HA41LJlS2VmZl7RnwULFqhFixYKDQ1Vt27dtHXr1orsDgAAMEyFgs+GDRuUnJyszz//XB6PRyUlJUpMTNT58+ftmgkTJuijjz7S8uXLtWHDBh07dkyPPfaYvby0tFRJSUkqLi7W5s2btXjxYmVmZmratGl2zeHDh5WUlKSHH35YeXl5Gj9+vEaPHq21a9faNcuWLVNqaqqmT5+uHTt2qH379nK73Tpx4sTNHA8AAFCDBViWZVV25a+//lqRkZHasGGDHnzwQZ09e1aNGzfW0qVL9bOf/UyStH//frVp00Y5OTnq3r27Pv74Y/30pz/VsWPHFBUVJUlauHChpkyZoq+//lohISGaMmWKVq1apb1799rbGjJkiM6cOaM1a9ZIkrp166Z7771X8+fPlySVlZUpNjZWzzzzjJ5//vkb6n9BQYEiIiJ09uxZOZ3Oyh6GK5SUlGj16tWavDVIRaUBfmu3qh15Oam6uwD4DeMQNU2L51dVdxcqxBFkaXbXUvXv31/BwcF+bftmfn/XupkNnz17VpLUoEEDSVJubq5KSkqUkJBg17Ru3VrNmjWzg09OTo7atm1rhx5JcrvdGjdunPbt26eOHTsqJyfHp43ymvHjx0uSiouLlZubq7S0NHt5YGCgEhISlJOTc83+FhUVqaioyJ4uKCiQ9N0bZElJSSWPwpXK23IEVjpTVgt/HgOgujEOUdM4gm6v13L52KuK1/TNtFnp4FNWVqbx48frvvvu0z333CNJ8nq9CgkJUb169Xxqo6Ki5PV67ZrLQ0/58vJl16spKCjQt99+q9OnT6u0tPSqNfv3779mnzMyMjRjxowr5mdlZSk8PPwG9rpiZnUp83ubVWn16tXV3QXA7xiHqClmd63uHlSOx+Pxe5sXLlyo9LqVDj7Jycnau3evNm3aVOmN32ppaWlKTU21pwsKChQbG6vExES/X+ryeDx6YXugispun1Pse9Pd1d0FwG8Yh6hp7klf+8NFPyKOQEuzupSpT58+VXKpq7IqFXxSUlK0cuVKbdy4UU2bNrXnR0dHq7i4WGfOnPE565Ofn6/o6Gi75vtPX5U/9XV5zfefBMvPz5fT6VRYWJiCgoIUFBR01ZryNq7G4XDI4XBcMT84ONjv/ymSVFQWcFvdW1AVxwCoboxD1BS30+v4clXxO/Zm2qvQU12WZSklJUUffPCB1q1bp7i4OJ/lnTt3VnBwsLKzs+15Bw4c0NGjR+VyuSRJLpdLe/bs8Xn6yuPxyOl0Kj4+3q65vI3ymvI2QkJC1LlzZ5+asrIyZWdn2zUAAADfV6EzPsnJyVq6dKk+/PBD1a1b174nJyIiQmFhYYqIiNCoUaOUmpqqBg0ayOl06plnnpHL5VL37t0lSYmJiYqPj9eTTz6p2bNny+v1aurUqUpOTrbPxowdO1bz58/X5MmT9Ytf/ELr1q3T+++/r1Wr/u+O9tTUVA0fPlxdunRR165dNXfuXJ0/f14jR47017EBAAA1TIWCz9tvvy1J6tmzp8/8RYsWacSIEZKk119/XYGBgRo0aJCKiorkdrv11ltv2bVBQUFauXKlxo0bJ5fLpdq1a2v48OGaOXOmXRMXF6dVq1ZpwoQJmjdvnpo2bap33nlHbvf/XfsePHiwvv76a02bNk1er1cdOnTQmjVrrrjhGQAAoFyFgs+NfORPaGioFixYoAULFlyzpnnz5j/45ELPnj21c+fO69akpKQoJSXlB/sEAAAg8V1dAADAIAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGBUOPhs3btSAAQMUExOjgIAArVixwmf5iBEjFBAQ4PPTt29fn5pTp05p2LBhcjqdqlevnkaNGqXCwkKfmt27d+uBBx5QaGioYmNjNXv27Cv6snz5crVu3VqhoaFq27atVq9eXdHdAQAABqlw8Dl//rzat2+vBQsWXLOmb9++On78uP3z+9//3mf5sGHDtG/fPnk8Hq1cuVIbN27U008/bS8vKChQYmKimjdvrtzcXP3qV79Senq6fvOb39g1mzdv1hNPPKFRo0Zp586dGjhwoAYOHKi9e/dWdJcAAIAhalV0hX79+qlfv37XrXE4HIqOjr7qsi+//FJr1qzRtm3b1KVLF0nSm2++qf79++vVV19VTEyMlixZouLiYr377rsKCQnR3Xffrby8PL322mt2QJo3b5769u2rSZMmSZJmzZolj8ej+fPna+HChRXdLQAAYIAKB58bsX79ekVGRqp+/frq1auXXnzxRTVs2FCSlJOTo3r16tmhR5ISEhIUGBioLVu26NFHH1VOTo4efPBBhYSE2DVut1uvvPKKTp8+rfr16ysnJ0epqak+23W73VdcertcUVGRioqK7OmCggJJUklJiUpKSvyx63Z7kuQItPzW5q3gz2MAVDfGIWoaR9Dt9VouH3tV8Zq+mTb9Hnz69u2rxx57THFxcTp06JB++ctfql+/fsrJyVFQUJC8Xq8iIyN9O1Grlho0aCCv1ytJ8nq9iouL86mJioqyl9WvX19er9eed3lNeRtXk5GRoRkzZlwxPysrS+Hh4ZXa3+uZ1aXM721WJe6RQk3EOERNMbtrdfegcjwej9/bvHDhQqXX9XvwGTJkiP3vtm3bql27drrjjju0fv169e7d29+bq5C0tDSfs0QFBQWKjY1VYmKinE6n37ZTUlIij8ejF7YHqqgswG/tVrW96e7q7gLgN4xD1DT3pK+t7i5UiCPQ0qwuZerTp4+Cg4P92nb5FZvKqJJLXZf7t3/7NzVq1EhfffWVevfurejoaJ04ccKn5tKlSzp16pR9X1B0dLTy8/N9asqnf6jmWvcWSd/de+RwOK6YHxwc7Pf/FEkqKgtQUent84ZbFccAqG6MQ9QUt9Pr+HJV8Tv2Ztqr8s/x+de//qWTJ0+qSZMmkiSXy6UzZ84oNzfXrlm3bp3KysrUrVs3u2bjxo0+1/A8Ho9atWql+vXr2zXZ2dk+2/J4PHK5XFW9SwAA4DZV4eBTWFiovLw85eXlSZIOHz6svLw8HT16VIWFhZo0aZI+//xzHTlyRNnZ2XrkkUfUsmVLud3fnb5t06aN+vbtqzFjxmjr1q367LPPlJKSoiFDhigmJkaSNHToUIWEhGjUqFHat2+fli1bpnnz5vlcpnruuee0Zs0azZkzR/v371d6erq2b9+ulJQUPxwWAABQE1U4+Gzfvl0dO3ZUx44dJUmpqanq2LGjpk2bpqCgIO3evVv//u//rrvuukujRo1S586d9emnn/pcYlqyZIlat26t3r17q3///rr//vt9PqMnIiJCWVlZOnz4sDp37qz/+q//0rRp03w+66dHjx5aunSpfvOb36h9+/b64x//qBUrVuiee+65meMBAABqsArf49OzZ09Z1rUfqVu79odvvmrQoIGWLl163Zp27drp008/vW7N448/rscff/wHtwcAACDxXV0AAMAgBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYFQ4+Gzdu1IABAxQTE6OAgACtWLHCZ7llWZo2bZqaNGmisLAwJSQk6ODBgz41p06d0rBhw+R0OlWvXj2NGjVKhYWFPjW7d+/WAw88oNDQUMXGxmr27NlX9GX58uVq3bq1QkND1bZtW61evbqiuwMAAAxS4eBz/vx5tW/fXgsWLLjq8tmzZ+uNN97QwoULtWXLFtWuXVtut1sXL160a4YNG6Z9+/bJ4/Fo5cqV2rhxo55++ml7eUFBgRITE9W8eXPl5ubqV7/6ldLT0/Wb3/zGrtm8ebOeeOIJjRo1Sjt37tTAgQM1cOBA7d27t6K7BAAADFGroiv069dP/fr1u+oyy7I0d+5cTZ06VY888ogk6b333lNUVJRWrFihIUOG6Msvv9SaNWu0bds2denSRZL05ptvqn///nr11VcVExOjJUuWqLi4WO+++65CQkJ09913Ky8vT6+99podkObNm6e+fftq0qRJkqRZs2bJ4/Fo/vz5WrhwYaUOBgAAqNkqHHyu5/Dhw/J6vUpISLDnRUREqFu3bsrJydGQIUOUk5OjevXq2aFHkhISEhQYGKgtW7bo0UcfVU5Ojh588EGFhITYNW63W6+88opOnz6t+vXrKycnR6mpqT7bd7vdV1x6u1xRUZGKiors6YKCAklSSUmJSkpKbnb3beVtOQItv7V5K/jzGADVjXGImsYRdHu9lsvHXlW8pm+mTb8GH6/XK0mKiorymR8VFWUv83q9ioyM9O1ErVpq0KCBT01cXNwVbZQvq1+/vrxe73W3czUZGRmaMWPGFfOzsrIUHh5+I7tYIbO6lPm9zarEPVKoiRiHqClmd63uHlSOx+Pxe5sXLlyo9Lp+DT4/dmlpaT5niQoKChQbG6vExEQ5nU6/baekpEQej0cvbA9UUVmA39qtanvT3dXdBcBvGIeoae5JX1vdXagQR6ClWV3K1KdPHwUHB/u17fIrNpXh1+ATHR0tScrPz1eTJk3s+fn5+erQoYNdc+LECZ/1Ll26pFOnTtnrR0dHKz8/36emfPqHasqXX43D4ZDD4bhifnBwsN//UySpqCxARaW3zxtuVRwDoLoxDlFT3E6v48tVxe/Ym2nPr5/jExcXp+joaGVnZ9vzCgoKtGXLFrlcLkmSy+XSmTNnlJuba9esW7dOZWVl6tatm12zceNGn2t4Ho9HrVq1Uv369e2ay7dTXlO+HQAAgO+rcPApLCxUXl6e8vLyJH13Q3NeXp6OHj2qgIAAjR8/Xi+++KL+8pe/aM+ePXrqqacUExOjgQMHSpLatGmjvn37asyYMdq6das+++wzpaSkaMiQIYqJiZEkDR06VCEhIRo1apT27dunZcuWad68eT6XqZ577jmtWbNGc+bM0f79+5Wenq7t27crJSXl5o8KAACokSp8qWv79u16+OGH7enyMDJ8+HBlZmZq8uTJOn/+vJ5++mmdOXNG999/v9asWaPQ0FB7nSVLliglJUW9e/dWYGCgBg0apDfeeMNeHhERoaysLCUnJ6tz585q1KiRpk2b5vNZPz169NDSpUs1depU/fKXv9Sdd96pFStW6J577qnUgQAAADVfhYNPz549ZVnXfqQuICBAM2fO1MyZM69Z06BBAy1duvS622nXrp0+/fTT69Y8/vjjevzxx6/fYQAAgP+P7+oCAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADG8HvwSU9PV0BAgM9P69at7eUXL15UcnKyGjZsqDp16mjQoEHKz8/3aePo0aNKSkpSeHi4IiMjNWnSJF26dMmnZv369erUqZMcDodatmypzMxMf+8KAACoYarkjM/dd9+t48eP2z+bNm2yl02YMEEfffSRli9frg0bNujYsWN67LHH7OWlpaVKSkpScXGxNm/erMWLFyszM1PTpk2zaw4fPqykpCQ9/PDDysvL0/jx4zV69GitXbu2KnYHAADUELWqpNFatRQdHX3F/LNnz+q3v/2tli5dql69ekmSFi1apDZt2ujzzz9X9+7dlZWVpS+++EKffPKJoqKi1KFDB82aNUtTpkxRenq6QkJCtHDhQsXFxWnOnDmSpDZt2mjTpk16/fXX5Xa7q2KXAABADVAlwefgwYOKiYlRaGioXC6XMjIy1KxZM+Xm5qqkpEQJCQl2bevWrdWsWTPl5OSoe/fuysnJUdu2bRUVFWXXuN1ujRs3Tvv27VPHjh2Vk5Pj00Z5zfjx46/br6KiIhUVFdnTBQUFkqSSkhKVlJT4Yc9ltydJjkDLb23eCv48BkB1YxyipnEE3V6v5fKxVxWv6Ztp0+/Bp1u3bsrMzFSrVq10/PhxzZgxQw888ID27t0rr9erkJAQ1atXz2edqKgoeb1eSZLX6/UJPeXLy5ddr6agoEDffvutwsLCrtq3jIwMzZgx44r5WVlZCg8Pr9T+Xs+sLmV+b7MqrV69urq7APgd4xA1xeyu1d2DyvF4PH5v88KFC5Ve1+/Bp1+/fva/27Vrp27duql58+Z6//33rxlIbpW0tDSlpqba0wUFBYqNjVViYqKcTqfftlNSUiKPx6MXtgeqqCzAb+1Wtb3pXCZEzcE4RE1zT/rtdR+rI9DSrC5l6tOnj4KDg/3advkVm8qokktdl6tXr57uuusuffXVV+rTp4+Ki4t15swZn7M++fn59j1B0dHR2rp1q08b5U99XV7z/SfB8vPz5XQ6rxuuHA6HHA7HFfODg4P9/p8iSUVlASoqvX3ecKviGADVjXGImuJ2eh1frip+x95Me1X+OT6FhYU6dOiQmjRpos6dOys4OFjZ2dn28gMHDujo0aNyuVySJJfLpT179ujEiRN2jcfjkdPpVHx8vF1zeRvlNeVtAAAAXI3fg8/EiRO1YcMGHTlyRJs3b9ajjz6qoKAgPfHEE4qIiNCoUaOUmpqqv/71r8rNzdXIkSPlcrnUvXt3SVJiYqLi4+P15JNPateuXVq7dq2mTp2q5ORk+2zN2LFj9fe//12TJ0/W/v379dZbb+n999/XhAkT/L07AACgBvH7pa5//etfeuKJJ3Ty5Ek1btxY999/vz7//HM1btxYkvT6668rMDBQgwYNUlFRkdxut9566y17/aCgIK1cuVLjxo2Ty+VS7dq1NXz4cM2cOdOuiYuL06pVqzRhwgTNmzdPTZs21TvvvMOj7AAA4Lr8Hnz+8Ic/XHd5aGioFixYoAULFlyzpnnz5j/4ZEPPnj21c+fOSvURAACYie/qAgAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgDIIPAAAwBsEHAAAYg+ADAACMQfABAADGIPgAAABjEHwAAIAxCD4AAMAYBB8AAGAMgg8AADAGwQcAABiD4AMAAIxB8AEAAMYg+AAAAGMQfAAAgDEIPgAAwBgEHwAAYAyCDwAAMAbBBwAAGIPgAwAAjEHwAQAAxrjtg8+CBQvUokULhYaGqlu3btq6dWt1dwkAAPxI3dbBZ9myZUpNTdX06dO1Y8cOtW/fXm63WydOnKjurgEAgB+h2zr4vPbaaxozZoxGjhyp+Ph4LVy4UOHh4Xr33Xeru2sAAOBHqFZ1d6CyiouLlZubq7S0NHteYGCgEhISlJOTc9V1ioqKVFRUZE+fPXtWknTq1CmVlJT4rW8lJSW6cOGCapUEqrQswG/tVrWTJ09WdxcAv2Ecoqapdel8dXehQmqVWbpwoUwnT55UcHCwX9s+d+6cJMmyrIr3y689uYW++eYblZaWKioqymd+VFSU9u/ff9V1MjIyNGPGjCvmx8XFVUkfbzeN5lR3DwAwDlGTDK3i9s+dO6eIiIgKrXPbBp/KSEtLU2pqqj1dVlamU6dOqWHDhgoI8N9fhAUFBYqNjdU///lPOZ1Ov7UL4MYxDoHqVZVj0LIsnTt3TjExMRVe97YNPo0aNVJQUJDy8/N95ufn5ys6Ovqq6zgcDjkcDp959erVq6ouyul08oYLVDPGIVC9qmoMVvRMT7nb9ubmkJAQde7cWdnZ2fa8srIyZWdny+VyVWPPAADAj9Vte8ZHklJTUzV8+HB16dJFXbt21dy5c3X+/HmNHDmyursGAAB+hG7r4DN48GB9/fXXmjZtmrxerzp06KA1a9ZcccPzreZwODR9+vQrLqsBuHUYh0D1+rGOwQCrMs+CAQAA3IZu23t8AAAAKorgAwAAjEHwAQAAxiD4VLP09HR16NChursBoJLWr1+vgIAAnTlzprq7Avwo3OiYaNGihebOnXtL+nQ5gs8tFBAQoBUrVvjMmzhxos9nEQGoWkeOHFFAQIDy8vKquytAjdSjRw8dP37c/oDBzMzMq35Y8LZt2/T000/f4t7d5o+z1wR16tRRnTp1qrsbAL6nuLhYISEh1d0N4LYTEhJyzW9QuFzjxo1vQW+uZMQZn549e+rZZ5/V5MmT1aBBA0VHRys9Pd1efubMGY0ePVqNGzeW0+lUr169tGvXLp82XnzxRUVGRqpu3boaPXq0nn/+eZ9LVNu2bVOfPn3UqFEjRURE6KGHHtKOHTvs5S1atJAkPfroowoICLCnL7/UlZWVpdDQ0CtODz733HPq1auXPb1p0yY98MADCgsLU2xsrJ599lmdP397fWsvzHSzY/HQoUN65JFHFBUVpTp16ujee+/VJ5984rONq51ZrVevnjIzMyX935cSd+zYUQEBAerZs6ckacSIERo4cKBeeuklxcTEqFWrVpKk3/3ud+rSpYvq1q2r6OhoDR06VCdOnPDvgQFusZ49eyolJUUpKSmKiIhQo0aN9MILL9jfdn769Gk99dRTql+/vsLDw9WvXz8dPHjQXv8f//iHBgwYoPr166t27dq6++67tXr1akm+l7rWr1+vkSNH6uzZswoICFBAQIA95i+/1DV06FANHjzYp48lJSVq1KiR3nvvPUnffTtDRkaG4uLiFBYWpvbt2+uPf/xjhffdiOAjSYsXL1bt2rW1ZcsWzZ49WzNnzpTH45EkPf744zpx4oQ+/vhj5ebmqlOnTurdu7dOnTolSVqyZIleeuklvfLKK8rNzVWzZs309ttv+7R/7tw5DR8+XJs2bdLnn3+uO++8U/3799e5c+ckfReMJGnRokU6fvy4PX253r17q169evrTn/5kzystLdWyZcs0bNgwSd+98fft21eDBg3S7t27tWzZMm3atEkpKSn+P2hAFbiZsVhYWKj+/fsrOztbO3fuVN++fTVgwAAdPXr0hre/detWSdInn3yi48eP689//rO9LDs7WwcOHJDH49HKlSslfffmO2vWLO3atUsrVqzQkSNHNGLECD8dDaD6LF68WLVq1dLWrVs1b948vfbaa3rnnXckffeHwPbt2/WXv/xFOTk5sixL/fv3V0lJiSQpOTlZRUVF2rhxo/bs2aNXXnnlqlcvevTooblz58rpdOr48eM6fvy4Jk6ceEXdsGHD9NFHH6mwsNCet3btWl24cEGPPvqoJCkjI0PvvfeeFi5cqH379mnChAn6+c9/rg0bNlRsxy0DPPTQQ9b999/vM+/ee++1pkyZYn366aeW0+m0Ll686LP8jjvusH79619blmVZ3bp1s5KTk32W33fffVb79u2vuc3S0lKrbt261kcffWTPk2R98MEHPnXTp0/3aee5556zevXqZU+vXbvWcjgc1unTpy3LsqxRo0ZZTz/9tE8bn376qRUYGGh9++231+wP8GNws2Pxau6++27rzTfftKevNs4iIiKsRYsWWZZlWYcPH7YkWTt37vSpGT58uBUVFWUVFRVddx+2bdtmSbLOnTtnWZZl/fWvf7Uk2WMUuB089NBDVps2bayysjJ73pQpU6w2bdpYf/vb3yxJ1meffWYv++abb6ywsDDr/ffftyzLstq2bWulp6dfte3vj4lFixZZERERV9Q1b97cev311y3LsqySkhKrUaNG1nvvvWcvf+KJJ6zBgwdblmVZFy9etMLDw63Nmzf7tDFq1CjriSeeqNC+G3PGp127dj7TTZo00YkTJ7Rr1y4VFhaqYcOG9v02derU0eHDh3Xo0CFJ0oEDB9S1a1ef9b8/nZ+frzFjxujOO+9URESEnE6nCgsLK/SXqPRd6l2/fr2OHTsm6buzTUlJSfaNYbt27VJmZqZPX91ut8rKynT48OEKbQuoDjczFgsLCzVx4kS1adNG9erVU506dfTll19WeJxdS9u2ba+4ryc3N1cDBgxQs2bNVLduXT300EOS5LdtAtWle/fuCggIsKddLpcOHjyoL774QrVq1VK3bt3sZQ0bNlSrVq305ZdfSpKeffZZvfjii7rvvvs0ffp07d69+6b6UqtWLf3Hf/yHlixZIkk6f/68PvzwQ/tqx1dffaULFy6oT58+Pu8P7733nv3+cMPbuqme3kaCg4N9pgMCAlRWVqbCwkI1adJE69evv2Kdq92Ffi3Dhw/XyZMnNW/ePDVv3lwOh0Mul0vFxcUV6ue9996rO+64Q3/4wx80btw4ffDBB/a9CdJ3b/z/+Z//qWefffaKdZs1a1ahbQHV4WbG4sSJE+XxePTqq6+qZcuWCgsL089+9jOfcRYQEGDfp1Cu/PT8D6ldu7bP9Pnz5+V2u+V2u7VkyRI1btxYR48eldvtrvDYBmqS0aNHy+12a9WqVcrKylJGRobmzJmjZ555ptJtDhs2TA899JBOnDghj8ejsLAw9e3bV5LsS2CrVq3ST37yE5/1KvpdYMYEn2vp1KmTvF6vatWqZd9w/H2tWrXStm3b9NRTT9nzvn+Pzmeffaa33npL/fv3lyT985//1DfffONTExwcrNLS0h/s07Bhw7RkyRI1bdpUgYGBSkpK8unvF198oZYtW97oLgK3hRsZi5999plGjBhhX/MvLCzUkSNHfGoaN26s48eP29MHDx7UhQsX7OnyMzo3Mhb379+vkydP6uWXX1ZsbKwkafv27RXZLeBHa8uWLT7T5fenxsfH69KlS9qyZYt69OghSTp58qQOHDig+Ph4uz42NlZjx47V2LFjlZaWpv/5n/+5avAJCQm5ofHWo0cPxcbGatmyZfr444/1+OOP238oxcfHy+Fw6OjRo/ZZ18oy5lLXtSQkJMjlcmngwIHKysrSkSNHtHnzZv33f/+3/Qb3zDPP6Le//a0WL16sgwcP6sUXX9Tu3bt9ThHeeeed+t3vfqcvv/xSW7Zs0bBhwxQWFuazrRYtWig7O1ter1enT5++Zp+GDRumHTt26KWXXtLPfvYznzQ7ZcoUbd68WSkpKcrLy9PBgwf14YcfcnMzbns3MhbvvPNO/fnPf1ZeXp527dqloUOHqqyszKedXr16af78+dq5c6e2b9+usWPH+pxlioyMVFhYmNasWaP8/HydPXv2mn1q1qyZQkJC9Oabb+rvf/+7/vKXv2jWrFlVcwCAW+zo0aNKTU3VgQMH9Pvf/15vvvmmnnvuOd1555165JFHNGbMGG3atEm7du3Sz3/+c/3kJz/RI488IkkaP3681q5dq8OHD2vHjh3661//qjZt2lx1Oy1atFBhYaGys7P1zTff+Pwh8n1Dhw7VwoUL5fF47MtcklS3bl1NnDhREyZM0OLFi3Xo0CHt2LFDb775phYvXlyh/TY++AQEBGj16tV68MEHNXLkSN11110aMmSI/vGPfygqKkrSd0EkLS1NEydOVKdOnXT48GGNGDFCoaGhdju//e1vdfr0aXXq1ElPPvmknn32WUVGRvpsa86cOfJ4PIqNjVXHjh2v2aeWLVuqa9eu2r17t89/vPTd/REbNmzQ3/72Nz3wwAPq2LGjpk2bppiYGD8eFeDWu5Gx+Nprr6l+/frq0aOHBgwYILfbrU6dOvm0M2fOHMXGxuqBBx7Q0KFDNXHiRIWHh9vLa9WqpTfeeEO//vWvFRMTY7+RX03jxo2VmZmp5cuXKz4+Xi+//LJeffXVqjkAwC321FNP6dtvv1XXrl2VnJys5557zv5AwUWLFqlz58766U9/KpfLJcuytHr1avuPiNLSUiUnJ6tNmzbq27ev7rrrLr311ltX3U6PHj00duxYDR48WI0bN9bs2bOv2adhw4bpiy++0E9+8hPdd999PstmzZqlF154QRkZGfZ2V61aZX9ExY0KsL5/MRw3pE+fPoqOjtbvfve76u4KAAAV0rNnT3Xo0KFavjKiuhl/j8+NuHDhghYuXCi3262goCD9/ve/1yeffGJ/9ggAALg9EHxuQPkp+JdeekkXL15Uq1at9Kc//UkJCQnV3TUAAFABXOoCAADGMP7mZgAAYA6CDwAAMAbBBwAAGIPgAwAAjEHwAQAAxiD4AAAAYxB8AACAMQg+AADAGAQfAABgjP8HaHMWNKrk8iUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "reviews.sentiment.hist()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:19.770797Z",
     "iopub.status.busy": "2023-11-07T10:37:19.770470Z",
     "iopub.status.idle": "2023-11-07T10:37:19.775239Z",
     "shell.execute_reply": "2023-11-07T10:37:19.774271Z",
     "shell.execute_reply.started": "2023-11-07T10:37:19.770770Z"
    }
   },
   "outputs": [],
   "source": [
    "sentiment_map = {\n",
    "    'negative': 0,\n",
    "    'neautral': 1,\n",
    "    'positive': 2\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:19.776802Z",
     "iopub.status.busy": "2023-11-07T10:37:19.776447Z",
     "iopub.status.idle": "2023-11-07T10:37:19.853687Z",
     "shell.execute_reply": "2023-11-07T10:37:19.852798Z",
     "shell.execute_reply.started": "2023-11-07T10:37:19.776774Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>качество плохое пошив ужасный (горловина напер...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Товар отдали другому человеку, я не получила п...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ужасная синтетика! Тонкая, ничего общего с пре...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>товар не пришел, продавец продлил защиту без м...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Кофточка голая синтетика, носить не возможно.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Очень глубокие проймы</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Я недовольна заказом.Я вот одного не понимаю п...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>заказала размер s на от 64,об 94,начнем с того...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Заказ я сделала в июле. С тех пор посылка отсл...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Ужасное качество товара!</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review  sentiment\n",
       "0  качество плохое пошив ужасный (горловина напер...          0\n",
       "1  Товар отдали другому человеку, я не получила п...          0\n",
       "2  Ужасная синтетика! Тонкая, ничего общего с пре...          0\n",
       "3  товар не пришел, продавец продлил защиту без м...          0\n",
       "4      Кофточка голая синтетика, носить не возможно.          0\n",
       "5                              Очень глубокие проймы          0\n",
       "6  Я недовольна заказом.Я вот одного не понимаю п...          0\n",
       "7  заказала размер s на от 64,об 94,начнем с того...          0\n",
       "8  Заказ я сделала в июле. С тех пор посылка отсл...          0\n",
       "9                           Ужасное качество товара!          0"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews.sentiment = reviews.sentiment.apply(lambda x: sentiment_map[x])\n",
    "reviews.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:19.859161Z",
     "iopub.status.busy": "2023-11-07T10:37:19.858548Z",
     "iopub.status.idle": "2023-11-07T10:37:19.869919Z",
     "shell.execute_reply": "2023-11-07T10:37:19.868931Z",
     "shell.execute_reply.started": "2023-11-07T10:37:19.859132Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(13098, 2)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_sample = reviews.sample(frac=0.15, replace=True, random_state=42)\n",
    "reviews_sample.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:19.871826Z",
     "iopub.status.busy": "2023-11-07T10:37:19.871534Z",
     "iopub.status.idle": "2023-11-07T10:37:20.134006Z",
     "shell.execute_reply": "2023-11-07T10:37:20.133055Z",
     "shell.execute_reply.started": "2023-11-07T10:37:19.871781Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjAAAAGdCAYAAAAMm0nCAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoZUlEQVR4nO3df3RU9Z3/8VcSkglBJjHQJGQJMZYtEH4KlDC2KmhIpFkPrOwWWkRWQVdO8DRkV5RzlJ/dA1KRYk2lrUjoVkRoRVegkBEMrBJAAzkNaDnqpsUuJtlqIeHXZMjc7x9+5x7G8CMTZ8h8Js/HOTkw977vJ5/3fOZmXtyZITGWZVkCAAAwSGxnTwAAACBYBBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHG6dfYEwsXn8+nkyZPq2bOnYmJiOns6AACgHSzLUnNzszIzMxUbe+XrLFEbYE6ePKmsrKzOngYAAOiATz/9VH379r3i/qgNMD179pT05R3gdDpDNq7X61VFRYUKCgoUHx8fsnEjSbT3SH/mi/Yeo70/Kfp7pL+Oa2pqUlZWlv08fiVRG2D8Lxs5nc6QB5ikpCQ5nc6ofFBK0d8j/Zkv2nuM9v6k6O+R/r6+a739gzfxAgAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABinW2dPwFRDFu+Sp/Xqv+o7kvxpRVFnTwEAgJDhCgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcbp19gQAAOjqbnpie2dPISiOOEsrx3TuHLgCAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOHwKCYCxhizeJU9rTGdPo93+tKKos6cARA2uwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAY52sFmBUrVigmJkYlJSX2tgsXLqi4uFi9evXSDTfcoClTpqihoSHguBMnTqioqEhJSUlKS0vTY489posXLwbUVFZWauTIkXI4HOrfv7/Ky8u/zlQBAEAU6XCAee+99/SLX/xCw4YNC9g+b948vfnmm9qyZYv27t2rkydP6t5777X3t7a2qqioSC0tLdq/f782bNig8vJyLVy40K6pq6tTUVGRxo8fr5qaGpWUlGj27NnatWtXR6cLAACiSIcCzJkzZzR9+nT96le/0o033mhvP336tNatW6dnn31Wd955p0aNGqX169dr//79OnDggCSpoqJCH3zwgX7zm99oxIgRmjhxopYtW6aysjK1tLRIktauXaucnBytWrVKgwYN0ty5c/VP//RPWr16dQhaBgAApuvWkYOKi4tVVFSk/Px8/fjHP7a3V1dXy+v1Kj8/3942cOBA9evXT1VVVRo7dqyqqqo0dOhQpaen2zWFhYWaM2eOjh07pltuuUVVVVUBY/hrLn2p6qs8Ho88Ho99u6mpSZLk9Xrl9Xo70uZl+cdyxFohG/N6COY+8NeG8n6LJPRnvmg/D7vSGkZrj8H254gz67HsP/fCsX7tHTPoALNp0yYdPnxY7733Xpt99fX1SkhIUEpKSsD29PR01dfX2zWXhhf/fv++q9U0NTXp/Pnz6t69e5vvvXz5ci1ZsqTN9oqKCiUlJbW/wXZaNtoX8jHDaceOHUEf43a7wzCTyEF/5ov287ArrGG099je/laOCfNEwiQc63fu3Ll21QUVYD799FP96Ec/ktvtVmJiYocmFi4LFixQaWmpfbupqUlZWVkqKCiQ0+kM2ffxer1yu9166v1YeXwxIRs33I4uLmx3rb/HCRMmKD4+Poyz6hz0Z75oPw+70hpGa4/B9jdksVnv8XTEWlo22heW9fO/gnItQQWY6upqNTY2auTIkfa21tZW7du3T88//7x27dqllpYWnTp1KuAqTENDgzIyMiRJGRkZOnToUMC4/k8pXVrz1U8uNTQ0yOl0XvbqiyQ5HA45HI422+Pj48Nycnh8MfK0mvODsyP3Qbjuu0hBf+aL9vOwK6xhtPfY3v5MehxfKhzr197xgnoT71133aXa2lrV1NTYX6NHj9b06dPtv8fHx2v37t32McePH9eJEyfkcrkkSS6XS7W1tWpsbLRr3G63nE6ncnNz7ZpLx/DX+McAAABdW1BXYHr27KkhQ4YEbOvRo4d69eplb581a5ZKS0uVmpoqp9OpRx99VC6XS2PHjpUkFRQUKDc3VzNmzNDKlStVX1+vJ598UsXFxfYVlEceeUTPP/+85s+frwcffFB79uzR5s2btX379lD0DAAADNehTyFdzerVqxUbG6spU6bI4/GosLBQP//5z+39cXFx2rZtm+bMmSOXy6UePXpo5syZWrp0qV2Tk5Oj7du3a968eVqzZo369u2rF198UYWF7X8fBwAAiF5fO8BUVlYG3E5MTFRZWZnKysqueEx2dvY1340/btw4HTly5OtODwAARCF+FxIAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACME1SAeeGFFzRs2DA5nU45nU65XC79/ve/t/dfuHBBxcXF6tWrl2644QZNmTJFDQ0NAWOcOHFCRUVFSkpKUlpamh577DFdvHgxoKayslIjR46Uw+FQ//79VV5e3vEOAQBA1AkqwPTt21crVqxQdXW13n//fd15552aNGmSjh07JkmaN2+e3nzzTW3ZskV79+7VyZMnde+999rHt7a2qqioSC0tLdq/f782bNig8vJyLVy40K6pq6tTUVGRxo8fr5qaGpWUlGj27NnatWtXiFoGAACm6xZM8T333BNw+z/+4z/0wgsv6MCBA+rbt6/WrVunjRs36s4775QkrV+/XoMGDdKBAwc0duxYVVRU6IMPPtBbb72l9PR0jRgxQsuWLdPjjz+uxYsXKyEhQWvXrlVOTo5WrVolSRo0aJDeeecdrV69WoWFhSFqGwAAmCyoAHOp1tZWbdmyRWfPnpXL5VJ1dbW8Xq/y8/PtmoEDB6pfv36qqqrS2LFjVVVVpaFDhyo9Pd2uKSws1Jw5c3Ts2DHdcsstqqqqChjDX1NSUnLV+Xg8Hnk8Hvt2U1OTJMnr9crr9Xa0zTb8YzlirZCNeT0Ecx/4a0N5v0US+jNftJ+HXWkNo7XHYPtzxJn1WPafe+FYv/aOGXSAqa2tlcvl0oULF3TDDTdo69atys3NVU1NjRISEpSSkhJQn56ervr6eklSfX19QHjx7/fvu1pNU1OTzp8/r+7du192XsuXL9eSJUvabK+oqFBSUlKwbV7TstG+kI8ZTjt27Aj6GLfbHYaZRA76M1+0n4ddYQ2jvcf29rdyTJgnEibhWL9z5861qy7oADNgwADV1NTo9OnT+u1vf6uZM2dq7969QU8w1BYsWKDS0lL7dlNTk7KyslRQUCCn0xmy7+P1euV2u/XU+7Hy+GJCNm64HV3c/pff/D1OmDBB8fHxYZxV56A/80X7ediV1jBaewy2vyGLzXqfpyPW0rLRvrCsn/8VlGsJOsAkJCSof//+kqRRo0bpvffe05o1azR16lS1tLTo1KlTAVdhGhoalJGRIUnKyMjQoUOHAsbzf0rp0pqvfnKpoaFBTqfzildfJMnhcMjhcLTZHh8fH5aTw+OLkafVnB+cHbkPwnXfRQr6M1+0n4ddYQ2jvcf29mfS4/hS4Vi/9o73tf8fGJ/PJ4/Ho1GjRik+Pl67d++29x0/flwnTpyQy+WSJLlcLtXW1qqxsdGucbvdcjqdys3NtWsuHcNf4x8DAAAgqCswCxYs0MSJE9WvXz81Nzdr48aNqqys1K5du5ScnKxZs2aptLRUqampcjqdevTRR+VyuTR27FhJUkFBgXJzczVjxgytXLlS9fX1evLJJ1VcXGxfPXnkkUf0/PPPa/78+XrwwQe1Z88ebd68Wdu3bw999wAAwEhBBZjGxkbdf//9+uyzz5ScnKxhw4Zp165dmjBhgiRp9erVio2N1ZQpU+TxeFRYWKif//zn9vFxcXHatm2b5syZI5fLpR49emjmzJlaunSpXZOTk6Pt27dr3rx5WrNmjfr27asXX3yRj1ADAABbUAFm3bp1V92fmJiosrIylZWVXbEmOzv7mu/EHzdunI4cORLM1AAAQBfC70ICAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcYIKMMuXL9e3v/1t9ezZU2lpaZo8ebKOHz8eUHPhwgUVFxerV69euuGGGzRlyhQ1NDQE1Jw4cUJFRUVKSkpSWlqaHnvsMV28eDGgprKyUiNHjpTD4VD//v1VXl7esQ4BAEDUCSrA7N27V8XFxTpw4IDcbre8Xq8KCgp09uxZu2bevHl68803tWXLFu3du1cnT57Uvffea+9vbW1VUVGRWlpatH//fm3YsEHl5eVauHChXVNXV6eioiKNHz9eNTU1Kikp0ezZs7Vr164QtAwAAEzXLZjinTt3BtwuLy9XWlqaqqurdfvtt+v06dNat26dNm7cqDvvvFOStH79eg0aNEgHDhzQ2LFjVVFRoQ8++EBvvfWW0tPTNWLECC1btkyPP/64Fi9erISEBK1du1Y5OTlatWqVJGnQoEF65513tHr1ahUWFoaodQAAYKqgAsxXnT59WpKUmpoqSaqurpbX61V+fr5dM3DgQPXr109VVVUaO3asqqqqNHToUKWnp9s1hYWFmjNnjo4dO6ZbbrlFVVVVAWP4a0pKSq44F4/HI4/HY99uamqSJHm9Xnm93q/TZgD/WI5YK2RjXg/B3Af+2lDeb5GE/swX7edhV1rDaO0x2P4ccWY9lv3nXjjWr71jdjjA+Hw+lZSU6Dvf+Y6GDBkiSaqvr1dCQoJSUlICatPT01VfX2/XXBpe/Pv9+65W09TUpPPnz6t79+5t5rN8+XItWbKkzfaKigolJSV1rMmrWDbaF/Ixw2nHjh1BH+N2u8Mwk8hBf+aL9vOwK6xhtPfY3v5WjgnzRMIkHOt37ty5dtV1OMAUFxfr6NGjeueddzo6REgtWLBApaWl9u2mpiZlZWWpoKBATqczZN/H6/XK7Xbrqfdj5fHFhGzccDu6uP0vvfl7nDBhguLj48M4q85Bf+aL9vOwK61htPYYbH9DFpv1Hk9HrKVlo31hWT//KyjX0qEAM3fuXG3btk379u1T37597e0ZGRlqaWnRqVOnAq7CNDQ0KCMjw645dOhQwHj+TyldWvPVTy41NDTI6XRe9uqLJDkcDjkcjjbb4+Pjw3JyeHwx8rSa84OzI/dBuO67SEF/5ov287ArrGG099je/kx6HF8qHOvX3vGC+hSSZVmaO3eutm7dqj179ignJydg/6hRoxQfH6/du3fb244fP64TJ07I5XJJklwul2pra9XY2GjXuN1uOZ1O5ebm2jWXjuGv8Y8BAAC6tqCuwBQXF2vjxo1644031LNnT/s9K8nJyerevbuSk5M1a9YslZaWKjU1VU6nU48++qhcLpfGjh0rSSooKFBubq5mzJihlStXqr6+Xk8++aSKi4vtKyiPPPKInn/+ec2fP18PPvig9uzZo82bN2v79u0hbh8AAJgoqCswL7zwgk6fPq1x48apT58+9terr75q16xevVr/8A//oClTpuj2229XRkaGXnvtNXt/XFyctm3bpri4OLlcLt133326//77tXTpUrsmJydH27dvl9vt1vDhw7Vq1Sq9+OKLfIQaAABICvIKjGVd+2NeiYmJKisrU1lZ2RVrsrOzr/lu/HHjxunIkSPBTA8AAHQR/C4kAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGCfoALNv3z7dc889yszMVExMjF5//fWA/ZZlaeHCherTp4+6d++u/Px8ffTRRwE1X3zxhaZPny6n06mUlBTNmjVLZ86cCaj5wx/+oNtuu02JiYnKysrSypUrg+8OAABEpaADzNmzZzV8+HCVlZVddv/KlSv13HPPae3atTp48KB69OihwsJCXbhwwa6ZPn26jh07JrfbrW3btmnfvn16+OGH7f1NTU0qKChQdna2qqur9ZOf/ESLFy/WL3/5yw60CAAAok23YA+YOHGiJk6ceNl9lmXppz/9qZ588klNmjRJkvTrX/9a6enpev311zVt2jR9+OGH2rlzp9577z2NHj1akvSzn/1M3/ve9/TMM88oMzNTL7/8slpaWvTSSy8pISFBgwcPVk1NjZ599tmAoAMAALqmoAPM1dTV1am+vl75+fn2tuTkZOXl5amqqkrTpk1TVVWVUlJS7PAiSfn5+YqNjdXBgwf1j//4j6qqqtLtt9+uhIQEu6awsFBPP/20/va3v+nGG29s8709Ho88Ho99u6mpSZLk9Xrl9XpD1qN/LEesFbIxr4dg7gN/bSjvt0hCf+aL9vOwK61htPYYbH+OOLMey/5zLxzr194xQxpg6uvrJUnp6ekB29PT0+199fX1SktLC5xEt25KTU0NqMnJyWkzhn/f5QLM8uXLtWTJkjbbKyoqlJSU1MGOrmzZaF/IxwynHTt2BH2M2+0Ow0wiB/2ZL9rPw66whtHeY3v7WzkmzBMJk3Cs37lz59pVF9IA05kWLFig0tJS+3ZTU5OysrJUUFAgp9MZsu/j9Xrldrv11Pux8vhiQjZuuB1dXNjuWn+PEyZMUHx8fBhn1Tnoz3zRfh52pTWM1h6D7W/I4l3XYVah44i1tGy0Lyzr538F5VpCGmAyMjIkSQ0NDerTp4+9vaGhQSNGjLBrGhsbA467ePGivvjiC/v4jIwMNTQ0BNT4b/trvsrhcMjhcLTZHh8fH5aTw+OLkafVnB+cHbkPwnXfRQr6M1+0n4ddYQ2jvcf29mfS4/hS4Vi/9o4X0v8HJicnRxkZGdq9e7e9rampSQcPHpTL5ZIkuVwunTp1StXV1XbNnj175PP5lJeXZ9fs27cv4HUwt9utAQMGXPblIwAA0LUEHWDOnDmjmpoa1dTUSPryjbs1NTU6ceKEYmJiVFJSoh//+Mf6r//6L9XW1ur+++9XZmamJk+eLEkaNGiQ7r77bj300EM6dOiQ3n33Xc2dO1fTpk1TZmamJOmHP/yhEhISNGvWLB07dkyvvvqq1qxZE/ASEQAA6LqCfgnp/fff1/jx4+3b/lAxc+ZMlZeXa/78+Tp79qwefvhhnTp1St/97ne1c+dOJSYm2se8/PLLmjt3ru666y7FxsZqypQpeu655+z9ycnJqqioUHFxsUaNGqXevXtr4cKFfIQaAABI6kCAGTdunCzryh/3iomJ0dKlS7V06dIr1qSmpmrjxo1X/T7Dhg3Tf//3fwc7PQAA0AXwu5AAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgHAIMAAAwDgEGAAAYhwADAACMQ4ABAADGIcAAAADjEGAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYBwCDAAAMA4BBgAAGIcAAwAAjEOAAQAAxiHAAAAA4xBgAACAcQgwAADAOAQYAABgnIgOMGVlZbrpppuUmJiovLw8HTp0qLOnBAAAIkDEBphXX31VpaWlWrRokQ4fPqzhw4ersLBQjY2NnT01AADQySI2wDz77LN66KGH9MADDyg3N1dr165VUlKSXnrppc6eGgAA6GTdOnsCl9PS0qLq6motWLDA3hYbG6v8/HxVVVVd9hiPxyOPx2PfPn36tCTpiy++kNfrDdncvF6vzp07p27eWLX6YkI2brh9/vnn7a719/j5558rPj4+jLPqHPRnvmg/D7vSGkZrj8H21+3i2eswq9Dp5rN07pwvLOvX3NwsSbIs6+pzCOl3DZG//vWvam1tVXp6esD29PR0/fGPf7zsMcuXL9eSJUvabM/JyQnLHE3Te1VnzwAA5yGiyQ/DPH5zc7OSk5OvuD8iA0xHLFiwQKWlpfZtn8+nL774Qr169VJMTOj+hdbU1KSsrCx9+umncjqdIRs3kkR7j/RnvmjvMdr7k6K/R/rrOMuy1NzcrMzMzKvWRWSA6d27t+Li4tTQ0BCwvaGhQRkZGZc9xuFwyOFwBGxLSUkJ1xTldDqj8kF5qWjvkf7MF+09Rnt/UvT3SH8dc7UrL34R+SbehIQEjRo1Srt377a3+Xw+7d69Wy6XqxNnBgAAIkFEXoGRpNLSUs2cOVOjR4/WmDFj9NOf/lRnz57VAw880NlTAwAAnSxiA8zUqVP1f//3f1q4cKHq6+s1YsQI7dy5s80be683h8OhRYsWtXm5KppEe4/0Z75o7zHa+5Oiv0f6C78Y61qfUwIAAIgwEfkeGAAAgKshwAAAAOMQYAAAgHEIMAAAwDgEGEllZWW66aablJiYqLy8PB06dOiq9Vu2bNHAgQOVmJiooUOHaseOHQH7LcvSwoUL1adPH3Xv3l35+fn66KOPwtnCVQXT369+9SvddtttuvHGG3XjjTcqPz+/Tf2//Mu/KCYmJuDr7rvvDncbVxVMj+Xl5W3mn5iYGFBj8hqOGzeuTX8xMTEqKiqyayJpDfft26d77rlHmZmZiomJ0euvv37NYyorKzVy5Eg5HA71799f5eXlbWqCPa/DJdj+XnvtNU2YMEHf+MY35HQ65XK5tGvXroCaxYsXt1m/gQMHhrGLqwu2x8rKyss+Ruvr6wPqTF3Dy51fMTExGjx4sF0TSWu4fPlyffvb31bPnj2VlpamyZMn6/jx49c8rrOfC7t8gHn11VdVWlqqRYsW6fDhwxo+fLgKCwvV2Nh42fr9+/frBz/4gWbNmqUjR45o8uTJmjx5so4ePWrXrFy5Us8995zWrl2rgwcPqkePHiosLNSFCxeuV1u2YPurrKzUD37wA7399tuqqqpSVlaWCgoK9L//+78BdXfffbc+++wz++uVV165Hu1cVrA9Sl/+75GXzv/Pf/5zwH6T1/C1114L6O3o0aOKi4vTP//zPwfURcoanj17VsOHD1dZWVm76uvq6lRUVKTx48erpqZGJSUlmj17dsCTfEceE+ESbH/79u3ThAkTtGPHDlVXV2v8+PG65557dOTIkYC6wYMHB6zfO++8E47pt0uwPfodP348oIe0tDR7n8lruGbNmoC+Pv30U6WmprY5ByNlDffu3avi4mIdOHBAbrdbXq9XBQUFOnv2yr9gMiKeC60ubsyYMVZxcbF9u7W11crMzLSWL19+2frvf//7VlFRUcC2vLw861//9V8ty7Isn89nZWRkWD/5yU/s/adOnbIcDof1yiuvhKGDqwu2v6+6ePGi1bNnT2vDhg32tpkzZ1qTJk0K9VQ7LNge169fbyUnJ19xvGhbw9WrV1s9e/a0zpw5Y2+LtDX0k2Rt3br1qjXz58+3Bg8eHLBt6tSpVmFhoX37695n4dKe/i4nNzfXWrJkiX170aJF1vDhw0M3sRBqT49vv/22Jcn629/+dsWaaFrDrVu3WjExMdaf/vQne1skr2FjY6Mlydq7d+8VayLhubBLX4FpaWlRdXW18vPz7W2xsbHKz89XVVXVZY+pqqoKqJekwsJCu76urk719fUBNcnJycrLy7vimOHSkf6+6ty5c/J6vUpNTQ3YXllZqbS0NA0YMEBz5szR559/HtK5t1dHezxz5oyys7OVlZWlSZMm6dixY/a+aFvDdevWadq0aerRo0fA9khZw2Bd6xwMxX0WSXw+n5qbm9ucgx999JEyMzN18803a/r06Tpx4kQnzbDjRowYoT59+mjChAl699137e3Rtobr1q1Tfn6+srOzA7ZH6hqePn1akto85i4VCc+FXTrA/PWvf1Vra2ub/903PT29zWuxfvX19Vet9/8ZzJjh0pH+vurxxx9XZmZmwIPw7rvv1q9//Wvt3r1bTz/9tPbu3auJEyeqtbU1pPNvj470OGDAAL300kt644039Jvf/EY+n0+33nqr/vKXv0iKrjU8dOiQjh49qtmzZwdsj6Q1DNaVzsGmpiadP38+JI/7SPLMM8/ozJkz+v73v29vy8vLU3l5uXbu3KkXXnhBdXV1uu2229Tc3NyJM22/Pn36aO3atfrd736n3/3ud8rKytK4ceN0+PBhSaH52RUpTp48qd///vdtzsFIXUOfz6eSkhJ95zvf0ZAhQ65YFwnPhRH7qwTQ+VasWKFNmzapsrIy4E2u06ZNs/8+dOhQDRs2TN/85jdVWVmpu+66qzOmGhSXyxXwS0FvvfVWDRo0SL/4xS+0bNmyTpxZ6K1bt05Dhw7VmDFjArabvoZdxcaNG7VkyRK98cYbAe8PmThxov33YcOGKS8vT9nZ2dq8ebNmzZrVGVMNyoABAzRgwAD79q233qpPPvlEq1ev1n/+53924sxCb8OGDUpJSdHkyZMDtkfqGhYXF+vo0aOd+p6q9urSV2B69+6tuLg4NTQ0BGxvaGhQRkbGZY/JyMi4ar3/z2DGDJeO9Of3zDPPaMWKFaqoqNCwYcOuWnvzzTerd+/e+vjjj7/2nIP1dXr0i4+P1y233GLPP1rW8OzZs9q0aVO7fhh25hoG60rnoNPpVPfu3UPymIgEmzZt0uzZs7V58+Y2l+q/KiUlRd/61reMWL8rGTNmjD3/aFlDy7L00ksvacaMGUpISLhqbSSs4dy5c7Vt2za9/fbb6tu371VrI+G5sEsHmISEBI0aNUq7d++2t/l8Pu3evTvgX+iXcrlcAfWS5Ha77fqcnBxlZGQE1DQ1NengwYNXHDNcOtKf9OU7x5ctW6adO3dq9OjR1/w+f/nLX/T555+rT58+IZl3MDra46VaW1tVW1trzz8a1lD68iOOHo9H99133zW/T2euYbCudQ6G4jHR2V555RU98MADeuWVVwI+/n4lZ86c0SeffGLE+l1JTU2NPf9oWEPpy0/3fPzxx+36R0RnrqFlWZo7d662bt2qPXv2KCcn55rHRMRzYUjeCmywTZs2WQ6HwyovL7c++OAD6+GHH7ZSUlKs+vp6y7Isa8aMGdYTTzxh17/77rtWt27drGeeecb68MMPrUWLFlnx8fFWbW2tXbNixQorJSXFeuONN6w//OEP1qRJk6ycnBzr/PnzEd/fihUrrISEBOu3v/2t9dlnn9lfzc3NlmVZVnNzs/Xv//7vVlVVlVVXV2e99dZb1siRI62///u/ty5cuHDd++tIj0uWLLF27dplffLJJ1Z1dbU1bdo0KzEx0Tp27JhdY/Ia+n33u9+1pk6d2mZ7pK1hc3OzdeTIEevIkSOWJOvZZ5+1jhw5Yv35z3+2LMuynnjiCWvGjBl2/f/8z/9YSUlJ1mOPPWZ9+OGHVllZmRUXF2ft3LnTrrnWfRbJ/b388stWt27drLKysoBz8NSpU3bNv/3bv1mVlZVWXV2d9e6771r5+flW7969rcbGxuven2UF3+Pq1aut119/3froo4+s2tpa60c/+pEVGxtrvfXWW3aNyWvod99991l5eXmXHTOS1nDOnDlWcnKyVVlZGfCYO3funF0Tic+FXT7AWJZl/exnP7P69etnJSQkWGPGjLEOHDhg77vjjjusmTNnBtRv3rzZ+ta3vmUlJCRYgwcPtrZv3x6w3+fzWU899ZSVnp5uORwO66677rKOHz9+PVq5rGD6y87OtiS1+Vq0aJFlWZZ17tw5q6CgwPrGN75hxcfHW9nZ2dZDDz3UKT9ULhVMjyUlJXZtenq69b3vfc86fPhwwHgmr6FlWdYf//hHS5JVUVHRZqxIW0P/R2q/+uXvaebMmdYdd9zR5pgRI0ZYCQkJ1s0332ytX7++zbhXu8+up2D7u+OOO65ab1lffmy8T58+VkJCgvV3f/d31tSpU62PP/74+jZ2iWB7fPrpp61vfvObVmJiopWammqNGzfO2rNnT5txTV1Dy/ryI8Pdu3e3fvnLX152zEhaw8v1JingvIrE58KY/z95AAAAY3Tp98AAAAAzEWAAAIBxCDAAAMA4BBgAAGAcAgwAADAOAQYAABiHAAMAAIxDgAEAAMYhwAAAAOMQYAAAgHEIMAAAwDgEGAAAYJz/B+ZXzRGan5J0AAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "reviews_sample.sentiment.hist()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:20.135847Z",
     "iopub.status.busy": "2023-11-07T10:37:20.135417Z",
     "iopub.status.idle": "2023-11-07T10:37:20.168296Z",
     "shell.execute_reply": "2023-11-07T10:37:20.167182Z",
     "shell.execute_reply.started": "2023-11-07T10:37:20.135810Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'cuda'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = ('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:20.169925Z",
     "iopub.status.busy": "2023-11-07T10:37:20.169605Z",
     "iopub.status.idle": "2023-11-07T10:37:20.178189Z",
     "shell.execute_reply": "2023-11-07T10:37:20.177401Z",
     "shell.execute_reply.started": "2023-11-07T10:37:20.169897Z"
    }
   },
   "outputs": [],
   "source": [
    "pretrained_model_name_or_path = 'DeepPavlov/rubert-base-cased'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:20.179815Z",
     "iopub.status.busy": "2023-11-07T10:37:20.179430Z",
     "iopub.status.idle": "2023-11-07T10:37:20.196794Z",
     "shell.execute_reply": "2023-11-07T10:37:20.195856Z",
     "shell.execute_reply.started": "2023-11-07T10:37:20.179782Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_labels = reviews.sentiment.nunique()\n",
    "num_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:20.198267Z",
     "iopub.status.busy": "2023-11-07T10:37:20.197982Z",
     "iopub.status.idle": "2023-11-07T10:37:26.792876Z",
     "shell.execute_reply": "2023-11-07T10:37:26.791949Z",
     "shell.execute_reply.started": "2023-11-07T10:37:20.198234Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "347fb1bd86f74ec1a4aa523c38f89a95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)lve/main/config.json:   0%|          | 0.00/642 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1953c5dcc9044135bae4f307d669feac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/714M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at DeepPavlov/rubert-base-cased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3d10903a6d343d5b45043cd6d961014",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/1.65M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c3620a988b54ba488fd49ef01a7da0c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32ea8fe9afbc43dbbae629666a58ebea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/24.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = BertForSequenceClassification.from_pretrained(pretrained_model_name_or_path, \n",
    "                                                      num_labels=num_labels)\n",
    "tokenizer = BertTokenizer.from_pretrained(pretrained_model_name_or_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:26.794557Z",
     "iopub.status.busy": "2023-11-07T10:37:26.794181Z",
     "iopub.status.idle": "2023-11-07T10:37:32.145005Z",
     "shell.execute_reply": "2023-11-07T10:37:32.144021Z",
     "shell.execute_reply.started": "2023-11-07T10:37:26.794523Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(119547, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = model.to(device)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:32.146449Z",
     "iopub.status.busy": "2023-11-07T10:37:32.146145Z",
     "iopub.status.idle": "2023-11-07T10:37:32.153357Z",
     "shell.execute_reply": "2023-11-07T10:37:32.152500Z",
     "shell.execute_reply.started": "2023-11-07T10:37:32.146423Z"
    }
   },
   "outputs": [],
   "source": [
    "embedding_parameters = [param for name, param in model.named_parameters() \n",
    "                        if 'classifier' not in name]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:32.154802Z",
     "iopub.status.busy": "2023-11-07T10:37:32.154529Z",
     "iopub.status.idle": "2023-11-07T10:37:34.832778Z",
     "shell.execute_reply": "2023-11-07T10:37:34.831828Z",
     "shell.execute_reply.started": "2023-11-07T10:37:32.154777Z"
    }
   },
   "outputs": [],
   "source": [
    "for param in embedding_parameters:\n",
    "    param.requires_grad = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:34.834947Z",
     "iopub.status.busy": "2023-11-07T10:37:34.834162Z",
     "iopub.status.idle": "2023-11-07T10:37:34.844604Z",
     "shell.execute_reply": "2023-11-07T10:37:34.843679Z",
     "shell.execute_reply.started": "2023-11-07T10:37:34.834913Z"
    }
   },
   "outputs": [],
   "source": [
    "X = reviews_sample.review\n",
    "y = reviews_sample.sentiment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:34.846042Z",
     "iopub.status.busy": "2023-11-07T10:37:34.845748Z",
     "iopub.status.idle": "2023-11-07T10:37:34.858692Z",
     "shell.execute_reply": "2023-11-07T10:37:34.857812Z",
     "shell.execute_reply.started": "2023-11-07T10:37:34.846009Z"
    }
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:34.860174Z",
     "iopub.status.busy": "2023-11-07T10:37:34.859896Z",
     "iopub.status.idle": "2023-11-07T10:37:34.867045Z",
     "shell.execute_reply": "2023-11-07T10:37:34.866202Z",
     "shell.execute_reply.started": "2023-11-07T10:37:34.860150Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[PAD]\n"
     ]
    }
   ],
   "source": [
    "tokenizer.pad_token_id = 0\n",
    "print(tokenizer.pad_token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:34.868930Z",
     "iopub.status.busy": "2023-11-07T10:37:34.868252Z",
     "iopub.status.idle": "2023-11-07T10:37:34.879263Z",
     "shell.execute_reply": "2023-11-07T10:37:34.878410Z",
     "shell.execute_reply.started": "2023-11-07T10:37:34.868897Z"
    }
   },
   "outputs": [],
   "source": [
    "class SentimentAnalysisDataset(Dataset):\n",
    "    def __init__(self, X, y, tokenizer=None, num_labels=3):\n",
    "        self.X = tokenizer(X.tolist(), \n",
    "                           truncation=True, \n",
    "                           padding=True,\n",
    "                           return_tensors=\"pt\",\n",
    "                           max_length=512).to(device)\n",
    "        self.y = torch.tensor(y.to_numpy(), dtype=torch.int64).to(device)\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.y)\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        return self.X.input_ids[idx], self.X.attention_mask[idx], self.y[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:34.880616Z",
     "iopub.status.busy": "2023-11-07T10:37:34.880335Z",
     "iopub.status.idle": "2023-11-07T10:37:49.839023Z",
     "shell.execute_reply": "2023-11-07T10:37:49.838206Z",
     "shell.execute_reply.started": "2023-11-07T10:37:34.880585Z"
    }
   },
   "outputs": [],
   "source": [
    "train_dataset = SentimentAnalysisDataset(X_train, y_train, tokenizer)\n",
    "test_dataset = SentimentAnalysisDataset(X_test, y_test, tokenizer)\n",
    "\n",
    "free_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:49.840622Z",
     "iopub.status.busy": "2023-11-07T10:37:49.840263Z",
     "iopub.status.idle": "2023-11-07T10:37:49.845155Z",
     "shell.execute_reply": "2023-11-07T10:37:49.844253Z",
     "shell.execute_reply.started": "2023-11-07T10:37:49.840590Z"
    }
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_epochs = 15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:49.846665Z",
     "iopub.status.busy": "2023-11-07T10:37:49.846366Z",
     "iopub.status.idle": "2023-11-07T10:37:49.857481Z",
     "shell.execute_reply": "2023-11-07T10:37:49.856488Z",
     "shell.execute_reply.started": "2023-11-07T10:37:49.846640Z"
    }
   },
   "outputs": [],
   "source": [
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:49.858866Z",
     "iopub.status.busy": "2023-11-07T10:37:49.858608Z",
     "iopub.status.idle": "2023-11-07T10:37:49.871266Z",
     "shell.execute_reply": "2023-11-07T10:37:49.870395Z",
     "shell.execute_reply.started": "2023-11-07T10:37:49.858841Z"
    }
   },
   "outputs": [],
   "source": [
    "optimizer = AdamW(model.classifier.parameters(), lr=0.01, weight_decay=1e-4)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:29:39.865354Z",
     "iopub.status.busy": "2023-11-07T11:29:39.864996Z",
     "iopub.status.idle": "2023-11-07T11:29:39.873173Z",
     "shell.execute_reply": "2023-11-07T11:29:39.872246Z",
     "shell.execute_reply.started": "2023-11-07T11:29:39.865325Z"
    }
   },
   "outputs": [],
   "source": [
    "def evaluate_model(model, data_loader):\n",
    "    ground_truth = []\n",
    "    predictions = []\n",
    "\n",
    "    model.eval()\n",
    "    for input_ids, attention_mask, labels in data_loader:\n",
    "        input_ids = input_ids.to(device)\n",
    "        attention_mask = attention_mask.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        _, indices = torch.max(outputs.logits, 1)\n",
    "        predictions.extend(indices.tolist())\n",
    "        ground_truth.extend(labels.tolist())\n",
    "        del input_ids\n",
    "        del attention_mask\n",
    "        del labels\n",
    "        del outputs\n",
    "        del indices\n",
    "        free_memory()\n",
    "    accuracy = accuracy_score(ground_truth, predictions)\n",
    "    f1 = f1_score(ground_truth, predictions, average='macro')\n",
    "    del predictions\n",
    "    del ground_truth\n",
    "    free_memory()\n",
    "    return accuracy, f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:49.889145Z",
     "iopub.status.busy": "2023-11-07T10:37:49.888809Z",
     "iopub.status.idle": "2023-11-07T10:37:49.899376Z",
     "shell.execute_reply": "2023-11-07T10:37:49.898567Z",
     "shell.execute_reply.started": "2023-11-07T10:37:49.889119Z"
    }
   },
   "outputs": [],
   "source": [
    "def train_model(model, train_loader, criterion, optimizer, num_epochs, \n",
    "                step_size=1, gamma=0.5):\n",
    "    scheduler = StepLR(optimizer, step_size=step_size, gamma=gamma)\n",
    "    loss_history = []\n",
    "    train_history = []\n",
    "    \n",
    "    for epoch in range(num_epochs):\n",
    "        model.train()\n",
    "        \n",
    "        loss_accum = 0\n",
    "        correct_samples = 0\n",
    "        total_samples = 0\n",
    "        \n",
    "        for i_step, (input_ids, attention_mask, y) in enumerate(train_loader):\n",
    "            input_ids = input_ids.to(device)\n",
    "            attention_mask = attention_mask.to(device)\n",
    "            y = y.to(device)\n",
    "            outputs = model(input_ids=input_ids, attention_mask=attention_mask)\n",
    "            #logits = torch.mean(outputs.logits, dim=1)\n",
    "            optimizer.zero_grad()\n",
    "            loss = criterion(outputs.logits, y)\n",
    "            loss.backward()\n",
    "            clip_grad_norm_(model.classifier.parameters(), 1.0)\n",
    "            optimizer.step()\n",
    "            \n",
    "            _, indices = torch.max(outputs.logits, 1)\n",
    "            correct_samples += torch.sum(indices == y)\n",
    "            total_samples += y.shape[0]\n",
    "            loss_accum += loss\n",
    "            \n",
    "            del outputs\n",
    "            del indices\n",
    "            del input_ids\n",
    "            del attention_mask\n",
    "            del y\n",
    "            free_memory()\n",
    "          \n",
    "        scheduler.step()\n",
    "        ave_loss = loss_accum / i_step\n",
    "        train_accuracy = float(correct_samples) / total_samples\n",
    "\n",
    "        loss_history.append(float(ave_loss))\n",
    "        train_history.append(train_accuracy)\n",
    "        \n",
    "        free_memory()\n",
    "\n",
    "        print(\"Average loss: %f, Train accuracy: %f\" % (ave_loss, train_accuracy))\n",
    "        \n",
    "    return loss_history, train_history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T10:37:49.900587Z",
     "iopub.status.busy": "2023-11-07T10:37:49.900321Z",
     "iopub.status.idle": "2023-11-07T11:28:42.420882Z",
     "shell.execute_reply": "2023-11-07T11:28:42.419857Z",
     "shell.execute_reply.started": "2023-11-07T10:37:49.900563Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average loss: 0.987085, Train accuracy: 0.524814\n",
      "Average loss: 0.893772, Train accuracy: 0.582458\n",
      "Average loss: 0.870194, Train accuracy: 0.598015\n",
      "Average loss: 0.857395, Train accuracy: 0.612044\n",
      "Average loss: 0.852505, Train accuracy: 0.609658\n",
      "Average loss: 0.848166, Train accuracy: 0.610708\n",
      "Average loss: 0.844660, Train accuracy: 0.618343\n",
      "Average loss: 0.844936, Train accuracy: 0.622161\n",
      "Average loss: 0.847967, Train accuracy: 0.624547\n",
      "Average loss: 0.842989, Train accuracy: 0.617293\n",
      "Average loss: 0.845643, Train accuracy: 0.623783\n",
      "Average loss: 0.847046, Train accuracy: 0.620920\n",
      "Average loss: 0.840913, Train accuracy: 0.622065\n",
      "Average loss: 0.846919, Train accuracy: 0.621684\n",
      "Average loss: 0.843450, Train accuracy: 0.618152\n"
     ]
    }
   ],
   "source": [
    "loss_history, train_history = train_model(model, \n",
    "                                          train_loader, \n",
    "                                          criterion, \n",
    "                                          optimizer, num_epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:28:42.423145Z",
     "iopub.status.busy": "2023-11-07T11:28:42.422836Z",
     "iopub.status.idle": "2023-11-07T11:28:42.680875Z",
     "shell.execute_reply": "2023-11-07T11:28:42.679928Z",
     "shell.execute_reply.started": "2023-11-07T11:28:42.423118Z"
    }
   },
   "outputs": [],
   "source": [
    "free_memory()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:28:42.682283Z",
     "iopub.status.busy": "2023-11-07T11:28:42.682008Z",
     "iopub.status.idle": "2023-11-07T11:28:42.976693Z",
     "shell.execute_reply": "2023-11-07T11:28:42.975581Z",
     "shell.execute_reply.started": "2023-11-07T11:28:42.682258Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGzCAYAAAD9pBdvAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABIBklEQVR4nO3deXwTdf4/8NfkTo+k9wW9hHJDQZAu4Cr8rFZgEXQ9cJFLcT1AhbIssMrlxYoXKniwurjryeqiogKKFVRYFAHxK0LL3SK9KSQ90zSZ3x/TpA30ps2nx+v5eMwjyWQm854Qklc/85nPSLIsyyAiIiISRCW6ACIiIuraGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaI6JLFxcVhxowZossgog6KYYSILvLmm29CkiTs3bu3zudHjx6NAQMGXNI2Nm/ejOXLl1/SaxBR58AwQkSXLCMjA//4xz+atc7mzZuxYsWKNqqIiDoShhEiumR6vR5arVZ0GXA6naioqBBdBhE1E8MIEV2yC/uM2O12rFixAgkJCTAYDAgODsaVV16Jbdu2AQBmzJiBtWvXAgAkSXJPLqWlpZg/fz6io6Oh1+vRu3dvPPPMM7jwIuOSJGHOnDl455130L9/f+j1emzZsgVxcXGYOHHiRXVWVFTAbDbjnnvuaYN3gYhaSiO6ACJqvywWCwoLCy+ab7fbG1xv+fLlWLlyJWbNmoXhw4fDarVi79692L9/P6699lrcc889yM7OxrZt2/DWW295rCvLMm644QZs374dd911FwYPHowvvvgCCxYswJkzZ/D88897LP/111/jP//5D+bMmYOQkBDEx8fjjjvuwKpVq1BUVISgoCD3sp9++imsVivuuOOOS3hXiKjVyUREF1i/fr0MoMGpf//+7uVjY2Pl6dOnux8nJibK48ePb3Abs2fPluv6Cvr4449lAPLjjz/uMf/mm2+WJUmSjx075p4HQFapVPKvv/7qsWxGRoYMQH7llVc85t9www1yXFyc7HQ6G30PiMh7eJiGiOq1du1abNu27aJp0KBBDa4XEBCAX3/9FUePHm32Njdv3gy1Wo0HH3zQY/78+fMhyzK2bNniMf/qq69Gv379POb16tULSUlJeOedd9zzioqKsGXLFkyZMsXjkBARicfDNERUr+HDh2PYsGEXzQ8MDKzz8I3Lo48+iokTJ6JXr14YMGAArr/+ekydOrXREAMAmZmZiIqKgr+/v8f8vn37up+vLT4+vs7XmTZtGubMmYPMzEzExsbigw8+gN1ux9SpUxutgYi8iy0jRNTqrrrqKhw/fhz//Oc/MWDAALz++uu4/PLL8frrr7f6toxGY53zJ0+eDK1W624defvttzFs2DD07t271WsgokvDMEJEbSIoKAgzZ87Ee++9h9OnT2PQoEEeg5zVd6gkNjYW2dnZKC4u9pifnp7ufr6p2x8/fjzeeecdZGZmYteuXWwVIWqnGEaIqNWdPXvW47Gfnx969uwJm83mnufr6wsAOH/+vMey48aNg8PhwJo1azzmP//885AkCWPHjm1yHVOnTsWhQ4ewYMECqNVqTJ48uZl7QkTewD4jRNTq+vXrh9GjR2Po0KEICgrC3r178eGHH2LOnDnuZYYOHQoAePDBB5GSkuIOCxMmTMCYMWPw8MMP49SpU0hMTMSXX36JTz75BHPnzkWPHj2aXMf48eMRHByMDz74AGPHjkVYWFir7ysRXTqGESJqdQ8++CA2bdqEL7/8EjabDbGxsXj88cexYMEC9zI33XQTHnjgAbz//vt4++23IcsyJk+eDJVKhU2bNmHp0qXYsGED1q9fj7i4ODz99NOYP39+s+rQ6XS47bbb8PLLL/MQDVE7JsnyBUMaEhF1IvPmzcMbb7yB3Nxc+Pj4iC6HiOrAPiNE1GlVVFTg7bffxh//+EcGEaJ2jIdpiKjTyc/Px1dffYUPP/wQZ8+exUMPPSS6JCJqAMMIEXU6hw4dwpQpUxAWFoYXX3wRgwcPFl0SETWAfUaIiIhIKPYZISIiIqEYRoiIiEioDtFnxOl0Ijs7G/7+/rzaJhERUQchyzKKi4sRFRUFlar+9o8OEUays7MRHR0tugwiIiJqgdOnT6N79+71Pt8hwojrUuKnT5+GyWQSXA0RERE1hdVqRXR0tPt3vD4dIoy4Ds2YTCaGESIiog6msS4Wze7A+u2332LChAmIioqCJEn4+OOPG11nx44duPzyy6HX69GzZ0+8+eabzd0sERERdVLNDiOlpaVITEzE2rVrm7T8yZMnMX78eIwZMwYHDhzA3LlzMWvWLHzxxRfNLpaIiIg6n2Yfphk7dizGjh3b5OVfffVVxMfH49lnnwUA9O3bFzt37sTzzz+PlJSU5m6eiIiIOpk27zOye/duJCcne8xLSUnB3Llz613HZrPBZrO5H1ut1ka343A4YLfbW1wndT5arRZqtVp0GURE1Ig2DyO5ubkIDw/3mBceHg6r1Yry8nIYjcaL1lm5ciVWrFjR5G2UlJTgt99+A0e2p9okSUL37t3h5+cnuhQiImpAuzybZvHixUhNTXU/dp0aVBeHw4HffvsNPj4+CA0N5aBoBEAZaKegoAC//fYbEhIS2EJCRNSOtXkYiYiIQF5ense8vLw8mEymOltFAECv10Ov1zfp9e12O2RZRmhoaL2vR11TaGgoTp06BbvdzjBCRNSOtfm1aUaMGIG0tDSPedu2bcOIESNadTtsEaEL8TNBRNQxNDuMlJSU4MCBAzhw4AAA5dTdAwcOICsrC4ByiGXatGnu5e+9916cOHECf/3rX5Geno6XX34Z//nPfzBv3rzW2QMiIiLq0JodRvbu3YshQ4ZgyJAhAIDU1FQMGTIES5cuBQDk5OS4gwkAxMfH4/PPP8e2bduQmJiIZ599Fq+//jpP6yUiIiIALegzMnr06AbPWqlrdNXRo0fjp59+au6myAtOnTqF+Ph4/PTTTxg8eHCdy7z55puYO3cuzp8/79XaiIioa2jzPiNUtxkzZmDSpEmiy2iS2267DUeOHGnSsm+++SYCAgLatiAiIupU2uWpvd4gyzKsFVUoKq1EbJAPVCp2dqyP0Wj0+plKDocDkiRBpWJeJiLq7DrdN70syyirrGrSdLKwBAXFFThzvrzJ6zQ0teaga9988w2GDx8OvV6PyMhILFq0CFVVVe7nP/zwQwwcOBBGoxHBwcFITk5GaWkpAOXChMOHD4evry8CAgIwatQoZGZmNri9EydOYMyYMfDx8UFiYiJ2797tfu7C1o6ff/4ZY8aMgb+/P0wmE4YOHYq9e/dix44dmDlzJiwWCyRJgiRJWL58OQDg3LlzmDZtGgIDA+Hj44OxY8fi6NGjF21j06ZN6NevH/R6PXbu3AmtVovc3FyPWufOnYvf//73LX1riYionel0LSPldgf6LRVzEb5Dj6bAR3fpb+mZM2cwbtw4zJgxA//+97+Rnp6Ou+++GwaDAcuXL0dOTg5uv/12rFq1CjfeeCOKi4vx3XffQZZlVFVVYdKkSbj77rvx3nvvobKyEnv27Gn0NNeHH34YzzzzDBISEvDwww/j9ttvx7Fjx6DRXLw/U6ZMwZAhQ/DKK69ArVbjwIED0Gq1GDlyJFavXo2lS5ciIyMDANyjn86YMQNHjx7Fpk2bYDKZsHDhQowbNw6HDh2CVqsFAJSVleGpp57C66+/juDgYERHR+Oyyy7DW2+9hQULFgBQxpV55513sGrVqkt+n4mIqH3odGGkM3j55ZcRHR2NNWvWQJIk9OnTB9nZ2Vi4cCGWLl2KnJwcVFVV4aabbkJsbCwAYODAgQCAoqIiWCwW/OEPf0CPHj0AKBcnbMxf/vIXjB8/HgCwYsUK9O/fH8eOHUOfPn0uWjYrKwsLFixwP5eQkOB+zmw2Q5IkREREuOe5QsiuXbswcuRIAMA777yD6OhofPzxx7jlllsAKEHj5ZdfRmJionvdu+66C+vXr3eHkU8//RQVFRW49dZbm/huEhFRe9fpwohRq8ahR5t+2nBhsQ251groNWr0DPO9pIGyjNrWGeXz8OHDGDFihEcto0aNcl+DJzExEddccw0GDhyIlJQUXHfddbj55psRGBiIoKAgzJgxAykpKbj22muRnJyMW2+9FZGRkQ1uc9CgQe77rmXz8/PrDCOpqamYNWsW3nrrLSQnJ+OWW25xB5/69kej0SApKck9Lzg4GL1798bhw4fd83Q6nUcdgNKi8sgjj+D777/H7373O7z55pu49dZb4evr2+D+EBFRx9Hp+oxIkgQfnabJU7dAH/jqNZAkoMohN2vdCydvjfipVquxbds2bNmyBf369cNLL72E3r174+TJkwCA9evXY/fu3Rg5ciQ2bNiAXr164fvvv2/wNV2HSoCakUudTmedyy5fvhy//vorxo8fj6+//hr9+vXDRx99dMn7ZTQaL3oPw8LCMGHCBKxfvx55eXnYsmUL7rzzzkveFhERtR+dLow0l1olIdhPuQ5OfrGtXVz5t2/fvti9e7dHLbt27YK/vz+6d+8OQAkMo0aNwooVK/DTTz9Bp9N5BIIhQ4Zg8eLF+N///ocBAwbg3XffbdUae/XqhXnz5uHLL7/ETTfdhPXr1wNQWjccDsdF+1NVVYUffvjBPe/s2bPIyMhAv379Gt3WrFmzsGHDBqxbtw49evTAqFGjWnVfiIhIrC4fRgAgxFcHlSSh3O5Aia2q8RVaicVicQ+t75pOnz6N+++/H6dPn8YDDzyA9PR0fPLJJ1i2bBlSU1OhUqnwww8/4Mknn8TevXuRlZWFjRs3oqCgAH379sXJkyexePFi7N69G5mZmfjyyy9x9OjRJvUbaYry8nLMmTMHO3bsQGZmJnbt2oUff/zR/fpxcXEoKSlBWloaCgsLUVZWhoSEBEycOBF33303du7ciZ9//hl33HEHunXrhokTJza6zZSUFJhMJjz++OOYOXNmq+wHERG1I3IHYLFYZACyxWK56Lny8nL50KFDcnl5+SVt48y5Mvnn0+fkY/nFl/Q6TTV9+nQZwEXTXXfdJcuyLO/YsUO+4oorZJ1OJ0dERMgLFy6U7Xa7LMuyfOjQITklJUUODQ2V9Xq93KtXL/mll16SZVmWc3Nz5UmTJsmRkZGyTqeTY2Nj5aVLl8oOh6POOk6ePCkDkH/66Sf3vHPnzskA5O3bt8uyLMvr16+XzWazLMuybLPZ5MmTJ8vR0dGyTqeTo6Ki5Dlz5ni8//fee68cHBwsA5CXLVsmy7IsFxUVyVOnTpXNZrNsNBrllJQU+ciRI+51am+jLkuWLJHVarWcnZ3d5Pe4tT4bRETUMg39ftcmyXI7OC7RCKvVCrPZDIvFApPJ5PFcRUUFTp48ifj4eBgMhhZvw17lRHpeMWRZRo9QP/jqO13f3g7trrvuQkFBATZt2tTkdVrrs0FERC3T0O93bfzFrabVqBDoo0VRaSUKim0MI+2ExWLBL7/8gnfffbdZQYSIiDoO9hmpJdRPDwmAtcKO8kpHo8tT25s4cSKuu+463Hvvvbj22mtFl0NERG2Af/7XoteqYTZqcb7cjoJiG2KCfUSX1OXt2LFDdAlERNTG2DJygVB/pW+BpbwSNjtbR4iIiNoaw8gFjDo1/A1ayAAKSmyiyyEiIur0GEbqEOavDIJ2rswOe1Xdo5ASERFR62AYqYOvXgNfvQayLLN1hIiIqI0xjNTD1TpSVFqJKgdbR4iIiNoKw0g9/PQaGLVqOGUZhaWVosshIiLqtBhG6iFJEkKrW0fOltjgcLb7gWoRFxeH1atXt+lrnDp1CpIk4cCBA5e0HSIiIheGkQaYjVroNWo4nDKKSluv74gkSQ1Oy5cvb9Hr/vjjj/jzn//canXWJTo6Gjk5ORgwYECjyzK4EBFRU3DQswa4Wkd+O1eGgpJKBPvqoVJJl/y6OTk57vsbNmzA0qVLkZGR4Z7n5+fnvi/LMhwOBzSaxv+pQkNDL7m2xqjVakRERLT5di5kt9uh1Wq9vl0iImp7na9lRJaBytJWmwI0ldA5K+CoKMG58+caXr6J1xyMiIhwT2azGZIkuR+np6fD398fW7ZswdChQ6HX67Fz504cP34cEydORHh4OPz8/HDFFVfgq6++8njdCw+xSJKE119/HTfeeCN8fHyQkJDQpOu7lJWV4c4774S/vz9iYmKwbt0693MXtnacO3cOU6ZMQWhoKIxGIxISErB+/XoAQHx8PABgyJAhkCQJo0ePBgA4nU48+uij6N69O/R6PQYPHoytW7detI0NGzbg6quvhsFgwLp162AymfDhhx961Prxxx/D19cXxcXFTXrviYio/el8LSP2MuDJqFZ7ORWAPk1d+G/ZgM63Vba7aNEiPPPMM7jssssQGBiI06dPY9y4cXjiiSeg1+vx73//GxMmTEBGRgZiYmLqfZ0VK1Zg1apVePrpp/HSSy9hypQpyMzMRFBQUL3rPPvss3jsscfwt7/9DR9++CHuu+8+XH311ejdu/dFyy5ZsgSHDh3Cli1bEBISgmPHjqG8vBwAsGfPHgwfPhxfffUV+vfvD51OBwB44YUX8Oyzz+K1117DkCFD8M9//hM33HADfv31VyQkJHi8B88++yyGDBkCg8GAn3/+GevXr8fNN9/sXsb12N/fv9nvMRERtQ+dr2Wkk3j00Udx7bXXokePHggKCkJiYiLuueceDBgwAAkJCXjsscfQo0ePRls6ZsyYgdtvvx09e/bEk08+iZKSEuzZs6fBdcaNG4f7778fPXv2xMKFCxESEoLt27fXuWxWVhaGDBmCYcOGIS4uDsnJyZgwYQKAmsNGwcHBiIiIcAegZ555BgsXLsTkyZPRu3dvPPXUUxg8ePBFHWfnzp2Lm266CfHx8YiMjMSsWbPwxRdfuA9z5efnY/PmzbjzzjsbfT+JiKj96nwtI1ofpYWileUXVyDPaoNBq0bPUF9IUh19R7Std2G9YcOGeTwuKSnB8uXL8fnnnyMnJwdVVVUoLy9HVlZWg68zaNAg931fX1+YTCbk5+c3eR3XIaT61rnvvvvwxz/+Efv378d1112HSZMmYeTIkfW+ttVqRXZ2NkaNGuUxf9SoUfj555895l34HgwfPhz9+/fHv/71LyxatAhvv/02YmNjcdVVVzW4P0RE1L51vpYRSVIOlbTyFBQYCJXOF+XQo9ipr3u5ugJKC/n6eh7u+ctf/oKPPvoITz75JL777jscOHAAAwcORGVlw2OgXNjpU5IkOJ0ND+LWnHXGjh2LzMxMzJs3D9nZ2bjmmmvwl7/8pcHXb6oL3wMAmDVrFt58800AyiGamTNn1h0MiYiow+h8YaSNaFQqBPkpfR7yi22Qm9hZtbXs2rULM2bMwI033oiBAwciIiICp06d8moN9QkNDcX06dPx9ttvY/Xq1e4Or64+Ig5HzdWPTSYToqKisGvXLo/X2LVrF/r169fotu644w5kZmbixRdfxKFDhzB9+vRW3BMiIhKh8x2maUMhfnoUllSirLIKpbYq+Bm8d6ppQkICNm7ciAkTJkCSJCxZsqTRFg5vWLp0KYYOHYr+/fvDZrPhs88+Q9++fQEAYWFhMBqN2Lp1K7p37w6DwQCz2YwFCxZg2bJl6NGjBwYPHoz169fjwIEDeOeddxrdXmBgIG666SYsWLAA1113Hbp3797Wu0hERG2MLSPNoFWrEORT0zriTc899xwCAwMxcuRITJgwASkpKbj88su9WkNddDodFi9ejEGDBuGqq66CWq3G+++/DwDQaDR48cUX8dprryEqKgoTJ04EADz44INITU3F/PnzMXDgQGzduhWbNm3yOJOmIXfddRcqKyvZcZWIqJOQZG8fb2gBq9UKs9kMi8UCk8nk8VxFRQVOnjyJ+Ph4GAyGNq+lssqBjNwSyJDRM8wPPjo2LnnbW2+95e6j4joUVBdvfzaIiMhTQ7/ftbFlpJl0GjUCfJTDMwVebh3p6srKynD8+HH8/e9/xz333NNgECEioo6DYaQFXBfQs5TbUWF3NLI0tZZVq1ahT58+iIiIwOLFi0WXQ0RErYRhpAUMWjVMBraOeNvy5ctht9uRlpbmcf0eIiLq2BhGWijMpLSOnC+zo7KKrSNEREQt1WnCiLf74froNPDTayBDRkFJwwOPkRgdoG82ERGhE4QRtVoNAI2ORNoWwqr7jpwrrYTdIX7MD/Lk+ky4PiNERNQ+dfjzUjUaDXx8fFBQUACtVguVynv5Si3L0EsOVNgdyCmSEebP00fbC6fTiYKCAvj4+ECj6fAfcyKiTq3Df0tLkoTIyEicPHkSmZmZXt9+ud2BsyWVKJSAYrMBKl4npd1QqVSIiYnhtWuIiNq5Dh9GAGUU0ISEBCGHapxOGXf/+0ecOluGO6+Mx5SkWK/XQHXT6XRebSkjIqKW6RRhBFD+ChY1yubNSZdh3oaf8fK3WfjTiJ4w6thHgYiIqKn4Z2MrmDAoCt0DjThbWokNP2aJLoeIiKhDYRhpBRq1Cvdc3QMAsO7bE6is4pk1RERETcUw0kpuGdodIX56ZFsq8MmBM6LLISIi6jAYRlqJQavG3b+PBwC88s1xOJwccIuIiKgpGEZa0ZTfxcJk0OBEQSm+/DVXdDlEREQdAsNIK/LTazBjZBwAYO2OYxyOnIiIqAkYRlrZjFHxMGrVOHjGiu+OFoouh4iIqN1jGGllQb463D48BgCwdvsxwdUQERG1fwwjbeDuq+KhVUv44WQR9mUWiS6HiIioXWMYaQORZiNuGtIdAPDy9uOCqyEiImrfGEbayL2je0AlAWnp+TicYxVdDhERUbvFMNJG4kN8MXZgJADglR1sHSEiIqoPw0gbun+0MkT8Z/+XjVOFpYKrISIiap8YRtpQ/ygzRvcOhVMGXvv2hOhyiIiI2iWGkTY2e0xPAMB/9/2GXEuF4GqIiIjaH4aRNnZFXBCuiAtEpcOJ179j6wgREdGFGEa84P7q1pF392ThXGml4GqIiIjaF4YRLxjdKxT9Ik0oq3Tgzf+dEl0OERFRu8Iw4gWSJOH+McqZNW/+7xRKbFWCKyIiImo/GEa8ZOyASMSH+MJSbsd7P2SJLoeIiKjdaFEYWbt2LeLi4mAwGJCUlIQ9e/bUu6zdbsejjz6KHj16wGAwIDExEVu3bm1xwR2VWiXh3qsvAwD847sTsFU5BFdERETUPjQ7jGzYsAGpqalYtmwZ9u/fj8TERKSkpCA/P7/O5R955BG89tpreOmll3Do0CHce++9uPHGG/HTTz9dcvEdzY1DuiPSbEB+sQ3/3XdGdDlERETtgiTLstycFZKSknDFFVdgzZo1AACn04no6Gg88MADWLRo0UXLR0VF4eGHH8bs2bPd8/74xz/CaDTi7bffbtI2rVYrzGYzLBYLTCZTc8ptd97YeRKPfXYIMUE++Hr+1dCoeaSMiIg6p6b+fjfrl7CyshL79u1DcnJyzQuoVEhOTsbu3bvrXMdms8FgMHjMMxqN2LlzZ73bsdlssFqtHlNncfvwaAT6aJFVVIbPf8kRXQ4REZFwzQojhYWFcDgcCA8P95gfHh6O3NzcOtdJSUnBc889h6NHj8LpdGLbtm3YuHEjcnLq/yFeuXIlzGaze4qOjm5Ome2aj06DO0fFA1AuoNfMhikiIqJOp82PEbzwwgtISEhAnz59oNPpMGfOHMycORMqVf2bXrx4MSwWi3s6ffp0W5fpVdNGxMFXp0Z6bjG+Tq+7rw0REVFX0awwEhISArVajby8PI/5eXl5iIiIqHOd0NBQfPzxxygtLUVmZibS09Ph5+eHyy67rN7t6PV6mEwmj6kzMftocceIWADA2u3H2DpCRERdWrPCiE6nw9ChQ5GWluae53Q6kZaWhhEjRjS4rsFgQLdu3VBVVYX//ve/mDhxYssq7iTuujIeOo0K+7PO4/0fT8PpZCAhIqKuqdmHaVJTU/GPf/wD//rXv3D48GHcd999KC0txcyZMwEA06ZNw+LFi93L//DDD9i4cSNOnDiB7777Dtdffz2cTif++te/tt5edEBh/gb8aXgMAGDxxl8wce0ufHe0gK0kRETU5Wiau8Jtt92GgoICLF26FLm5uRg8eDC2bt3q7tSalZXl0R+koqICjzzyCE6cOAE/Pz+MGzcOb731FgICAlptJzqqRWP7IMBHi398ewK/nLFg6ht7MLJHMP56fR8Mjg4QXR4REZFXNHucERE60zgjdTlbYsPa7cfx9veZqHQ4AQDX94/AX1J6oWeYv+DqiIiIWqapv98MI+3I6aIyrP7qKDb+9BtkGVBJwM1Du2Nuci9EBRhFl0dERNQsDCMdWEZuMZ75MgPbDilnLek0KkwfEYv7R/dEoK9OcHVERERNwzDSCezLLMJTWzKw51QRAMBfr8Gfr7oMd14ZD199s7v7EBEReRXDSCchyzJ2HCnAqq0ZOJyjDIsf4qfHg9f0xOQrYqDT8No2RETUPjGMdDJOp4xP/y8bz355BFlFZQCA6CAj5l/bGzckRkGlkgRXSERE5IlhpJOqrHJiw49ZeCHtGApLbACAPhH+WHh9H4zuHQpJYighIqL2gWGkkyurrML6Xafw6o7jKLZVAQCGxwVh4djeGBobJLg6IiIihpEu41xpJV755jje/N8pVFYpY5Qk9w3HgpTe6B3BMUqIiEgchpEuJsdSjhe+Oor/7D0NpwxIEnDjkG6Yl9wL0UE+ossjIqIuiGGkizqWX4LntmVg8y+5AACtWsKUpFjM+X89EeKnF1wdERF1JQwjXdzPp89j1Rfp2HXsLADAV6fGrN9fhlm/j4e/QSu4OiIi6goYRggAsPNoIZ7amo5fzlgAAEG+Oswe0xN3/C4Geo1acHVERNSZMYyQmyzL2HIwF898kYEThaUAgG4BRsy7thduHNINao5RQkREbYBhhC5S5XDig32/YfVXR5BnVcYoSQjzw7QRsRjdO4wdXYmIqFUxjFC9KuwO/Ot/p/DyjuOwlNvd8xPC/DCmTxhG9w7FsNggDjVPRESXhGGEGmUpt+O9PVn4+nA+9mWdg8NZ81Hw02twZc8QjOkTitG9wxBuMgislIiIOiKGEWoWS5kd3x0rwPb0AnxzJB+FJZUez/eLNGFMn1CM6R2GwdEB0KjZakJERA1jGKEWczplHMy2YHt6AbZn5OPn386j9qfEbNTi9wkhGNM7DFf3DuX4JUREVCeGEWo1Z0ts+Paoq9WkwKOfiSQBg7qZMbp3GMb0CcOgbmZeQZiIiAAwjFAbcThlHDh9DjsylFaTg2esHs8H++pwda9QjO4ThqsSQhDgoxNUKRERicYwQl6Rb63AjiMF2JGRj++OFLqvIAwAKgm4PCbQfYZOv0gTJImtJkREXQXDCHmd3eHEvsxz2J6Rjx3pBcjIK/Z4PsxfjzG9wzCmTyhG9QzhsPRERJ0cwwgJd+Z8OXZk5GN7egF2HStEud3hfk6jknBFXBDG9AnF4OhARJoNCDPpOUQ9EVEnwjBC7YqtyoE9J4vcfU1OFJTWuVyInx6RZgMizAZEmg2INBs9HoebDDBoGViIiDoChhFq1zLPlmJHhnJ2zomCEuRYKmCrcjZp3WBfnUdYcd2PMBsQVf2YgYWISDyGEepQZFnGuTI7cizlyLVUIMdSgRxLOXIsFcitnrIt5aiwNy2wBPpoEWE2IqpWq4rnYyOMOgYWIqK21NTfb40XayKqlyRJCPLVIchXh/5R5jqXkWUZlnK7O6Bk1woursc55ytQbnfgXJkd58rsOJxjrfO1AGXwNtehnzB/vXJr0iPMX49Qf0P1rZ6tLEREbYxhhDoMSZIQ4KNDgI8OfSPrTtiyLMNaUVVHWCmvbm2pQM75cpRWOmApt8NSbkd6bnGdr+ViNmoR5q+vDio1ISWsdojx18NXz/9OREQtwW9P6lQkSYLZqIXZqEXvCP96lyuusFcHlgrkWSqQX1yB/GIb8q22mvvFNlRWOd2h5Wh+SYPb9tWpEWYyKEHFvzq4mC6+bzZqOd4KEVEtDCPUJfkbtPA3aJEQXn9gkWUZ1vIqdzjJs14cWAqKbci3VqC00oHSSgdOFpbiZGHdZwq56DQqhPopLS3h/kp4cR2iCvTVIchHV+uxlqc7t5IqR02wPF99aylTbnUaFSLNBkQFKGdvcQwcIu9iGCGqhyRJMPtoYfZpOLQAQKmtqjqo1LSq5BdXoMBqQ15xRXWAscFSbkdllRNnzpfjzPnyJtXhq1MjyE8JKbXDSqArsPjoEOyn3Ab56mA2aqHupNcHkmUZJbYqJVCU2WGtDhbny1who1KZ53pcfWspt6Ok1ujAjfHTa5SztQJqOj1HmY2IDKg53byrHJaTZRlVThm2KidsdodyW+VEhet+rXm2Kgds9lr3q5yw2Z2ocM+vtax7PQeqHDIizAbEBvsgNtgXsUHKbZi/vtNd60qWZRSU2HC6qAyZZ5XpdFEZHLLs/j/s+n8e6KtV/jCpPjyt03Teq6XzbBoiL6qwO5TWlGIbCmq1rhSVVuJcWaVyW2rH2erHDmfz/3uqJCDAR4dAH22dYSXQR+cON0G+Oph9tFBJEpyyDFlWviydtW/hmg84ZbnWcspzzur5rnVlVC/nvHhd13Ny9fKudUsqqmq1VlR6tF7UDh2WcnuL3pPa/PUaJWQatQiovi2vdCDHUoHs8+WwVjQttJgMGndLSmSAEZGmmvASWT1fROdnp1NGaWUViiuqYK2wo7iiCsUVdljLq2895lfBWm5Hqa3KHQwq7J5BwlblwCW+5S1m0KoQE1QroIS4gooPugUYoVG3zx9n1x8cWUVlyDpbqoSOIiV0ZBWVoazS0fiL1MFfr0GgO6xoa4WWWv+3fXUI8tUisDrAiP7DhKf2EnVwrs6450orlXBSWomisurb6skVYFxTU39IOzqdRoUAo2egMBt17sc181zLKM+ZDJpGf8BKbVU1p5afrznNPLu683OOpaLJrSyBPlpEmo2ICqg5pTyqunUlymxEuNlz1GFZVlogrLXCQ32hwjXfWh0oXM8X26rQlt/qOrUKeq0Keo0aes0F9zUq6LW17mvUMLie19bMu3A9lSQhx1KOU64f7rNlOHO+vMHgqVFJ6BZorNWSUh1agn0QE+TT5kHQWmFHVnWtWUVlyCoqdd/PPl/eYICTJCDKbERMkFJrTLAP9BrVRX+UuP6/nyurbFEglCSlA74rsChh5YIQU30b7KtDZICh1Q8LM4wQdUF2hxPny+w4V1aJsyW1v9gq3a0t7i+8EmVeY4PNqSRAJUmQJOXQlYSax+75AFSq2s+5ngckSMrtBeuoql9LkpTDIubqwFA7ZJg8HteEDdGnWxdX2N0tKbXP0sq54BTzpgjx08FPr3GHC7ujdb6StWoJ/gYlgCl9pDQwVd/6G7QwGWvm++s1MGjVHgHB4BEylOd0apXXDpvYHU5kny/HqbNK68Kps67DGqXIKipr9HMbYXId9qkJKbFBvogN8YGpCX2CnE4ZecUVSsCoDhmZ1S0dWUVlOFdmb3B9V6tOTFBNQIoJ9kFskA+6BRqb9aPvdMqwVthrhRX7RX+c1PzfVpazlDdcX13eu/t3GNEjuNnrNYRhhIiapKL6R9MVEGqHBJ710zKuzs/ZlnL34H0552uCimtefT+okqQ0ybtDhLEmVHiEC2OtcFHr1mTUQq9Rddp/P6dTRn6xDafOliLrbJnSolKkBJXMs2UobqSFMNBHWxNQgn0RHWhEia3K3bKRebYUp8+Vo7KRwBPip0N0kBIwYoJ9qw8pKY9D/fVC3/8qhxPny+0XhBW7xx8o7jBTprTE/Pe+kQ2ehdgSDCNERO2Ya9Th7PPlKLc7PEKFr07T6TpueovrfXW1oJwqLENmUc3hn8ISW5Nfy3UoyHU4RWnh8HW3cvh1kU7Ml4IjsBIRtWO1Rx2m1lP7fR0SE3jR8yW2qurDLjWHfk4XlcHfoEFM9eGU2OpDK5FmQ7vtJNvZMIwQEVGX4afXoF+UCf2i2MrenjDyERERkVAMI0RERCQUwwgREREJxTBCREREQjGMEBERkVAMI0RERCQUwwgREREJxTBCREREQjGMEBERkVAMI0RERCQUwwgREREJxTBCREREQjGMEBERkVAMI0RERCQUwwgREREJxTBCREREQjGMEBERkVAMI0RERCQUwwgREREJxTBCREREQjGMEBERkVAMI0RERCQUwwgREREJxTBCREREQrUojKxduxZxcXEwGAxISkrCnj17Glx+9erV6N27N4xGI6KjozFv3jxUVFS0qGAiIiLqXJodRjZs2IDU1FQsW7YM+/fvR2JiIlJSUpCfn1/n8u+++y4WLVqEZcuW4fDhw3jjjTewYcMG/O1vf7vk4omIiKjjk2RZlpuzQlJSEq644gqsWbMGAOB0OhEdHY0HHngAixYtumj5OXPm4PDhw0hLS3PPmz9/Pn744Qfs3Lmzzm3YbDbYbDb3Y6vViujoaFgsFphMpuaUS0RERIJYrVaYzeZGf7+b1TJSWVmJffv2ITk5ueYFVCokJydj9+7dda4zcuRI7Nu3z30o58SJE9i8eTPGjRtX73ZWrlwJs9nsnqKjo5tTJhEREXUgmuYsXFhYCIfDgfDwcI/54eHhSE9Pr3OdP/3pTygsLMSVV14JWZZRVVWFe++9t8HDNIsXL0Zqaqr7satlhIiIiDqfNj+bZseOHXjyySfx8ssvY//+/di4cSM+//xzPPbYY/Wuo9frYTKZPCYiIiLqnJrVMhISEgK1Wo28vDyP+Xl5eYiIiKhznSVLlmDq1KmYNWsWAGDgwIEoLS3Fn//8Zzz88MNQqXh2MRERUVfWrCSg0+kwdOhQj86oTqcTaWlpGDFiRJ3rlJWVXRQ41Go1AKCZfWeJiIioE2pWywgApKamYvr06Rg2bBiGDx+O1atXo7S0FDNnzgQATJs2Dd26dcPKlSsBABMmTMBzzz2HIUOGICkpCceOHcOSJUswYcIEdyghIiKirqvZYeS2225DQUEBli5ditzcXAwePBhbt251d2rNysryaAl55JFHIEkSHnnkEZw5cwahoaGYMGECnnjiidbbCyIiIuqwmj3OiAhNPU+ZiIiI2o82GWeEiIiIqLUxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREK1KIysXbsWcXFxMBgMSEpKwp49e+pddvTo0ZAk6aJp/PjxLS6aiIiIOo9mh5ENGzYgNTUVy5Ytw/79+5GYmIiUlBTk5+fXufzGjRuRk5Pjng4ePAi1Wo1bbrnlkosnIiKijq/ZYeS5557D3XffjZkzZ6Jfv3549dVX4ePjg3/+8591Lh8UFISIiAj3tG3bNvj4+DCMEBEREYBmhpHKykrs27cPycnJNS+gUiE5ORm7d+9u0mu88cYbmDx5Mnx9fetdxmazwWq1ekxERETUOTUrjBQWFsLhcCA8PNxjfnh4OHJzcxtdf8+ePTh48CBmzZrV4HIrV66E2Wx2T9HR0c0pk4iIiDoQr55N88Ybb2DgwIEYPnx4g8stXrwYFovFPZ0+fdpLFRIREZG3aZqzcEhICNRqNfLy8jzm5+XlISIiosF1S0tL8f777+PRRx9tdDt6vR56vb45pREREVEH1ayWEZ1Oh6FDhyItLc09z+l0Ii0tDSNGjGhw3Q8++AA2mw133HFHyyolIiKiTqlZLSMAkJqaiunTp2PYsGEYPnw4Vq9ejdLSUsycORMAMG3aNHTr1g0rV670WO+NN97ApEmTEBwc3DqVExERUafQ7DBy2223oaCgAEuXLkVubi4GDx6MrVu3uju1ZmVlQaXybHDJyMjAzp078eWXX7ZO1URERNRpSLIsy6KLaIzVaoXZbIbFYoHJZBJdDhERETVBU3+/eW0aIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCEiIiKhGEaIiIhIKIYRIiIiEophhIiIiIRiGCGirkuWAXs54HSIroSoS9OILoCIqNkcVUBlMWBzTSXVt9aaeZV1zHMvV2u+7ADUeiCkFxDaGwjtU3MbFA+otaL3tvXJMlCSB5w9Dpw9BpTkA3p/wBgAGMyAofrW9VjrA0iS4KKpM2MYISLvkmWgshQoLQBKC5Vbd2CoKzjUETDsZa1bk8MG5P2iTLWptEBwz+pw0rsmpAT3BDT61q2hLZQVKYGjqDp0uMJH0QnlvWwqlbb+oFLX4wufU6lbf9+aylEFOCqVf2OHHaiyVT+u9LwPAJKqelJX30o181TqWs83MHksJ13wehcux4DnwjBCRJfOYa8JFrVDRn33q8pbZ7tqvfIXvd4f0PsBelPNY51f9X3TBcvUmqfzU+aVnQUKMoCC9Orb6sleChQcVqbaJLXSalK7FSW0NxCcAOh8WmffmspWUitsnKgOG9WPy8/Vv56kAgJigKAegClSCYjl54EKC1BRfVt+Xmk5ctpr/g1bQm9qOMgYTMqhsnoDg115rvZ993K177uWq6wJILKzZTV7iyucaH0B32DAxzWFAD5BgG9I3fP0pk4VZiRZlmXRRTTGarXCbDbDYrHAZDKJLoeo85Nl5QepdpAoya8/ZFScb/42NEbAL1T5gjUG1AoR/rXCQz2TrjpYtGXrhNMJWM/UCim1gorNUs9KkvID7xFS+gChvZS6W8peAZw7Watl43j1/eNASW7D6/pHAcE9qqeeSvgI7gkExjb+/rlasSrO1x1U6nvsum8vbfk+txW1XtlvtU6ZNNW3kJTgIjuqb53K/stOJSi557mWkT3n1V4GXvhZVWmaFlp8agUcAa15Tf39Zhgh6opsxUB+OpB/CCg8Uitg1AoZzqrmvaakUr4EfUOVL0HfUGXyC625X/s5nW/b7Ftbk2WgOLcmnBRWB5T8w0B5Uf3rmbpf3CcltBdgDFSed1QB5zPrOKxyHLCcRoM/cD4htcLGZcptcA/lvsj3uapSOczmDirn6g4yFVblx1WjU8JC7ZCg1te67woQF4aJJq6j0ninNUGWq6cLQ0zt0CJ7hhtbidJCV3YWKCtUbkvP1j2vpSFP599w60vCdYB/RKu+FQwjRKQ0XRceUX4o8w/V3J7Patr6enOtYBFSd6hwTcZAQNXFT9ArLbygFaX6tiSv/nX8wpXDReczGw6AelN1wOhREzZcj40Brb4r1I7Zy5X+QBeFlsKa8OIxr0gJPI258wsg5netWmpTf7/ZZ4SoM3A6gKKTnoEj/5DyV3V9X0J+EUBYX+UvdFPkBSEjTLntCJ002xPfEMD3SiDuSs/55eeAgiMXhxTrb9VBpTqsaIw1LRruwFF9aMU3pFP1EaBLoDUC5m7K1BROp3Jo8cLQUuoKL9XBxj+ybetuAMMIUUciy0o/BlfgyDtUc6ilqqLudQxmIKyfEjzC+tXc9wnybu1dmTEQiElSptoqrMq/nb1MCRz+kWxdotanUimfQWMggJ6iq6kTwwhRe1V6Fsj/9YJDLIeVY/B10RiVvgjh/auDR3X48I/kX9TtlcEEdB8mugoi4RhGyPscduDUTiD9M+DIF8pfhxedetncMyv8lQ5rHVHtzqS1g0dpft3LqzTKKaTulo7q4BEYJ3Y8ByKiFmIYIe+oLAOOfw0c/hQ4svXiU0HrPVWyGdxjTtQxvsRFY07UCj1qXfWYBPaasQ0avW+/YAyEJixb332nvf59Coy74BBLXyWIdNTgRURUB4YRajvl55SWj8OfAsfSPAe68gkB+owD+kxQBo9yj7pZ1zDedQzhXVlrnms0TocNKLMpHbE6Gldn0tqHWEJ6K6GJiKiTYxih1mXNUQ6/pH+mHIqpfaqiOQbo+weg7wQgOqn1Dim4r1NS4hlqbFbP0FLf5KisHrdAW2tcAq3nGAUXzavrvrbWGAiNLVvrvs5X6WRKRNRFMYzQpSs8BqR/Chz+DDiz1/O5sH5Anz8oISRiUNt0pFRravUUJyKijoZhhJpPloGcA0r4SP9MGTOhtu7DlfDR5w/KOAlEREQNaNEJ7WvXrkVcXBwMBgOSkpKwZ8+eBpc/f/48Zs+ejcjISOj1evTq1QubN29uUcEkiKMKOPkdsGUhsHogsG408N0zShBRaYAe/w8Y/xwwPwOYtQ0Y9RCDCBERNUmzW0Y2bNiA1NRUvPrqq0hKSsLq1auRkpKCjIwMhIWFXbR8ZWUlrr32WoSFheHDDz9Et27dkJmZiYCAgNaon9qSvQI4sV1pATmyRRmpz0XrA/RMVvp/JFzH4aiJiKjFmn1tmqSkJFxxxRVYs2YNAMDpdCI6OhoPPPAAFi1adNHyr776Kp5++mmkp6dDq9W2qEhem8aLKizA0W3KGTBHt3lekMkYCPQepxx+6TFGGZKYiIioHm1ybZrKykrs27cPixcvds9TqVRITk7G7t2761xn06ZNGDFiBGbPno1PPvkEoaGh+NOf/oSFCxdCra77bAqbzQabzeaxM9SGSvKB9M+V/h8nvvEc98LUDegzXgkgsaOUzqJEREStqFm/LIWFhXA4HAgPD/eYHx4ejvT09DrXOXHiBL7++mtMmTIFmzdvxrFjx3D//ffDbrdj2bJlda6zcuVKrFixojmlUXOVFACHPgYO/hfI+h4elycP6VVzBkzU5RxKnIiI2lSb/5nrdDoRFhaGdevWQa1WY+jQoThz5gyefvrpesPI4sWLkZqa6n5stVoRHR3d1qV2fhVWpQXklw+AEzs8r+YadXn1GTATgNBewkokIqKup1lhJCQkBGq1Gnl5eR7z8/LyEBERUec6kZGR0Gq1Hodk+vbti9zcXFRWVkKnu3hYa71eD72ely5vFVU2pe/HLx8ow7DXvrJr1OXAwJuBfhMBc3dxNRIRUZfWrDCi0+kwdOhQpKWlYdKkSQCUlo+0tDTMmTOnznVGjRqFd999F06nE6rqS2MfOXIEkZGRdQYRagVOhzL66S8fAIc2eV73JTgBGHiLEkJ46i0REbUDzT5Mk5qaiunTp2PYsGEYPnw4Vq9ejdLSUsycORMAMG3aNHTr1g0rV64EANx3331Ys2YNHnroITzwwAM4evQonnzySTz44IOtuyddnSwD2fuBXz4EDm4ESnJrnvOPAgbcpISQyET2ASEional2WHktttuQ0FBAZYuXYrc3FwMHjwYW7dudXdqzcrKcreAAEB0dDS++OILzJs3D4MGDUK3bt3w0EMPYeHCha23F11ZwRHg4IdKK0jRiZr5hgCg/yQlgMSMBFQtGt+OiIiozTV7nBEROM7IBSxngF83KgEk5+ea+RqjciXcgbcAPa7hZeaJiEioNhlnhAQqKwIOfaIchsncBfepuK6h2AfeogxIxkvOExFRB8Mw0p5VlgIZW5QAcuwrz8HIYkZWnwkzCfANFlYiERHRpWIYaW8cduD4duUQTPrnnsOxhw9UAsiAPwIBHHeFiIg6B4aR9sDpBE5/rwSQXz8GyotqnguMAwbcrISQsL6iKiQiImozDCMilZ8Ddq5WDsNYf6uZ7xuqtH4MvAXoNpSn4hIRUafGMCKKww68O1lpEQEAnT/Q7walBSTuKl6QjoiIugz+4ony1XIliOhNwA0vAr2uB7RG0VURERF5HcOICIc+AXavUe5PehnoO0FsPURERAJxWE5vO3sc+Hi2cn/EHAYRIiLq8hhGvKmyDNgwFagsBmJGAMnLRVdEREQkHMOIt8gy8Pl8IP9XwDcMuHk9oNaKroqIiEg4hhFv2f8v4Od3AUkF3PxPwBQpuiIiIqJ2gWHEG7IPAJv/qtz/f0uA+N8LLYeIiKg9YRhpa+XngP9MAxw25fTdUXNFV0RERNSuMIy0JacT+Og+4HwmEBAL3PgqoOJbTkREVBt/GdvSrtXAkS2AWg/c+m/AGCi6IiIionaHYaStnPwW+Pox5f64VUDUYKHlEBERtVcMI23BmgN8eCcgO4HEPwGXTxddERERUbvFMNLaHHYliJQWAGH9gfHP8qq7REREDWAYaW1pK4Cs/ylX4b3134DOR3RFRERE7RrDSGs6/Cnwv5eU+5NeBkJ6iq2HiIioA2AYaS1njwMf36/cHzEH6HeD2HqIiIg6CIaR1mAvVwY2s1l5ATwiIqJmYhhpDZ//Bcg7CPiG8gJ4REREzcQwcqn2/xs48LZyAbw/vsEL4BERETUTw8ilyPlZaRUBgDEPA5ddLbYeIiKiDohhpKXKz3teAO/KVNEVERERdUgMIy0hy8qZM+dOAQExvAAeERHRJeAvaEvsegHI+BxQ63gBPCIiokvEMNJcp3Yqo6wCwNhVQNQQsfUQERF1cAwjzVGcC3wwU7kA3qDJwNAZoisiIiLq8BhGmspRVX0BvHwgrB/wh+d5ATwiIqJWwDDSVF8/CmTuqr4A3lu8AB4REVErYRhpivTPlU6rADBxDS+AR0RE1IoYRhpTdAL46D7l/u9mA/0nCS2HiIios2EYaYj7AngWIDoJuHaF6IqIiIg6HYaRhmxeAOT+AviEALe8yQvgERERtQGGkfr89Dbw01vKBfBufgMwRYmuiIiIqFNiGKlLzv8Bn89X7o/5G3DZaKHlEBERdWYMIxdyXQCvqgJIuA64cr7oioiIiDo1hpHaZBn4ZDZw7iRgjgFufI0XwCMiImpj/KWt7X8vAemfVV8A71+AT5DoioiIiDo9hhGXU7uAr5Yr96//O9DtcqHlEBERdRUMIwBQnAd8OBOQHcCg24Bhd4quiIiIqMtgGHFUAf+9CyjJA0L78gJ4REREXsYwsv1x4NR3gM4PuO0tQOcruiIiIqIupWuHkfTNwM7nlfsT1wAhCWLrISIi6oK6bhipLAM2PaDcT7oP6H+j2HqIiIi6qK4bRnQ+wOR3gX4TgWsfFV0NERFRl6URXYBQMUnKRERERMJ03ZYRIiIiahcYRoiIiEgohhEiIiISimGEiIiIhGIYISIiIqEYRoiIiEgohhEiIiISimGEiIiIhGIYISIiIqEYRoiIiEgohhEiIiISimGEiIiIhGIYISIiIqE6xFV7ZVkGAFitVsGVEBERUVO5frddv+P16RBhpLi4GAAQHR0tuBIiIiJqruLiYpjN5nqfl+TG4ko74HQ6kZ2dDX9/f0iS1Gqva7VaER0djdOnT8NkMrXa63YkXf096Or7D/A94P537f0H+B605f7Lsozi4mJERUVBpaq/Z0iHaBlRqVTo3r17m72+yWTqkh/A2rr6e9DV9x/ge8D979r7D/A9aKv9b6hFxIUdWImIiEgohhEiIiISqkuHEb1ej2XLlkGv14suRZiu/h509f0H+B5w/7v2/gN8D9rD/neIDqxERETUeXXplhEiIiISj2GEiIiIhGIYISIiIqEYRoiIiEgohhEiIiISqkuHkbVr1yIuLg4GgwFJSUnYs2eP6JK8YuXKlbjiiivg7++PsLAwTJo0CRkZGaLLEubvf/87JEnC3LlzRZfiVWfOnMEdd9yB4OBgGI1GDBw4EHv37hVdllc4HA4sWbIE8fHxMBqN6NGjBx577LFGL+bVkX377beYMGECoqKiIEkSPv74Y4/nZVnG0qVLERkZCaPRiOTkZBw9elRMsW2koffAbrdj4cKFGDhwIHx9fREVFYVp06YhOztbXMGtrLHPQG333nsvJEnC6tWrvVJblw0jGzZsQGpqKpYtW4b9+/cjMTERKSkpyM/PF11am/vmm28we/ZsfP/999i2bRvsdjuuu+46lJaWii7N63788Ue89tprGDRokOhSvOrcuXMYNWoUtFottmzZgkOHDuHZZ59FYGCg6NK84qmnnsIrr7yCNWvW4PDhw3jqqaewatUqvPTSS6JLazOlpaVITEzE2rVr63x+1apVePHFF/Hqq6/ihx9+gK+vL1JSUlBRUeHlSttOQ+9BWVkZ9u/fjyVLlmD//v3YuHEjMjIycMMNNwiotG009hlw+eijj/D9998jKirKS5UBkLuo4cOHy7Nnz3Y/djgcclRUlLxy5UqBVYmRn58vA5C/+eYb0aV4VXFxsZyQkCBv27ZNvvrqq+WHHnpIdEles3DhQvnKK68UXYYw48ePl++8806PeTfddJM8ZcoUQRV5FwD5o48+cj92Op1yRESE/PTTT7vnnT9/Xtbr9fJ7770noMK2d+F7UJc9e/bIAOTMzEzvFOVF9e3/b7/9Jnfr1k0+ePCgHBsbKz///PNeqadLtoxUVlZi3759SE5Ods9TqVRITk7G7t27BVYmhsViAQAEBQUJrsS7Zs+ejfHjx3t8DrqKTZs2YdiwYbjlllsQFhaGIUOG4B//+Ifosrxm5MiRSEtLw5EjRwAAP//8M3bu3ImxY8cKrkyMkydPIjc31+P/gtlsRlJSUpf8TnSxWCyQJAkBAQGiS/EKp9OJqVOnYsGCBejfv79Xt90hrtrb2goLC+FwOBAeHu4xPzw8HOnp6YKqEsPpdGLu3LkYNWoUBgwYILocr3n//fexf/9+/Pjjj6JLEeLEiRN45ZVXkJqair/97W/48ccf8eCDD0Kn02H69Omiy2tzixYtgtVqRZ8+faBWq+FwOPDEE09gypQpoksTIjc3FwDq/E50PdfVVFRUYOHChbj99tu7zJV8n3rqKWg0Gjz44INe33aXDCNUY/bs2Th48CB27twpuhSvOX36NB566CFs27YNBoNBdDlCOJ1ODBs2DE8++SQAYMiQITh48CBeffXVLhFG/vOf/+Cdd97Bu+++i/79++PAgQOYO3cuoqKiusT+U8PsdjtuvfVWyLKMV155RXQ5XrFv3z688MIL2L9/PyRJ8vr2u+RhmpCQEKjVauTl5XnMz8vLQ0REhKCqvG/OnDn47LPPsH37dnTv3l10OV6zb98+5Ofn4/LLL4dGo4FGo8E333yDF198ERqNBg6HQ3SJbS4yMhL9+vXzmNe3b19kZWUJqsi7FixYgEWLFmHy5MkYOHAgpk6dinnz5mHlypWiSxPC9b3X1b8TgZogkpmZiW3btnWZVpHvvvsO+fn5iImJcX8vZmZmYv78+YiLi2vz7XfJMKLT6TB06FCkpaW55zmdTqSlpWHEiBECK/MOWZYxZ84cfPTRR/j6668RHx8vuiSvuuaaa/DLL7/gwIED7mnYsGGYMmUKDhw4ALVaLbrENjdq1KiLTuc+cuQIYmNjBVXkXWVlZVCpPL/+1Go1nE6noIrEio+PR0REhMd3otVqxQ8//NAlvhNdXEHk6NGj+OqrrxAcHCy6JK+ZOnUq/u///s/jezEqKgoLFizAF1980ebb77KHaVJTUzF9+nQMGzYMw4cPx+rVq1FaWoqZM2eKLq3NzZ49G++++y4++eQT+Pv7u48Jm81mGI1GwdW1PX9//4v6x/j6+iI4OLjL9JuZN28eRo4ciSeffBK33nor9uzZg3Xr1mHdunWiS/OKCRMm4IknnkBMTAz69++Pn376Cc899xzuvPNO0aW1mZKSEhw7dsz9+OTJkzhw4ACCgoIQExODuXPn4vHHH0dCQgLi4+OxZMkSREVFYdKkSeKKbmUNvQeRkZG4+eabsX//fnz22WdwOBzu78agoCDodDpRZbeaxj4DF4YvrVaLiIgI9O7du+2L88o5O+3USy+9JMfExMg6nU4ePny4/P3334suySsA1DmtX79edGnCdLVTe2VZlj/99FN5wIABsl6vl/v06SOvW7dOdEleY7Va5YceekiOiYmRDQaDfNlll8kPP/ywbLPZRJfWZrZv317n//vp06fLsqyc3rtkyRI5PDxc1uv18jXXXCNnZGSILbqVNfQenDx5st7vxu3bt4suvVU09hm4kDdP7ZVkuRMPOUhERETtXpfsM0JERETtB8MIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREIxjBAREZFQDCNEREQkFMMIERERCcUwQkREREL9f73JBlH9WqsYAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"History\")\n",
    "plt.plot(loss_history, label=\"Loss history\")\n",
    "plt.plot(train_history, label=\"Train history\")\n",
    "plt.legend(loc=\"upper left\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:28:42.978782Z",
     "iopub.status.busy": "2023-11-07T11:28:42.978087Z",
     "iopub.status.idle": "2023-11-07T11:28:42.983330Z",
     "shell.execute_reply": "2023-11-07T11:28:42.982433Z",
     "shell.execute_reply.started": "2023-11-07T11:28:42.978744Z"
    }
   },
   "outputs": [],
   "source": [
    "test_loader = DataLoader(test_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:29:45.586234Z",
     "iopub.status.busy": "2023-11-07T11:29:45.585415Z",
     "iopub.status.idle": "2023-11-07T11:30:31.914870Z",
     "shell.execute_reply": "2023-11-07T11:30:31.913950Z",
     "shell.execute_reply.started": "2023-11-07T11:29:45.586201Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test accuracy is 0.6515267175572519\n",
      "Test f1 score is 0.6463834249748095\n"
     ]
    }
   ],
   "source": [
    "accuracy, f1 = evaluate_model(model, test_loader)\n",
    "print(f\"Test accuracy is {accuracy}\")\n",
    "print(f\"Test f1 score is {f1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-11-07T11:31:43.343091Z",
     "iopub.status.busy": "2023-11-07T11:31:43.342431Z",
     "iopub.status.idle": "2023-11-07T11:31:44.679307Z",
     "shell.execute_reply": "2023-11-07T11:31:44.678501Z",
     "shell.execute_reply.started": "2023-11-07T11:31:43.343061Z"
    }
   },
   "outputs": [],
   "source": [
    "torch.save(model, 'st_rubert-sentiment-classifier.pth')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
